You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by to...@apache.org on 2014/08/14 17:20:22 UTC

[3/6] Implemented New rolling shard algorithm. This should allow eventual shard consistency between all clients without the need for an external locking allocation system

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e9d652dd/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardedEdgeSerializationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardedEdgeSerializationImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardedEdgeSerializationImpl.java
new file mode 100644
index 0000000..c33b835
--- /dev/null
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardedEdgeSerializationImpl.java
@@ -0,0 +1,656 @@
+/*
+ *
+ *  * Licensed to the Apache Software Foundation (ASF) under one
+ *  * or more contributor license agreements.  See the NOTICE file
+ *  * distributed with this work for additional information
+ *  * regarding copyright ownership.  The ASF licenses this file
+ *  * to you under the Apache License, Version 2.0 (the
+ *  * "License"); you may not use this file except in compliance
+ *  * with the License.  You may obtain a copy of the License at
+ *  *
+ *  *    http://www.apache.org/licenses/LICENSE-2.0
+ *  *
+ *  * Unless required by applicable law or agreed to in writing,
+ *  * software distributed under the License is distributed on an
+ *  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ *  * KIND, either express or implied.  See the License for the
+ *  * specific language governing permissions and limitations
+ *  * under the License.
+ *
+ */
+package org.apache.usergrid.persistence.graph.serialization.impl.shard.impl;
+
+
+import java.util.Iterator;
+import java.util.UUID;
+
+import javax.inject.Inject;
+
+import org.apache.usergrid.persistence.core.astyanax.CassandraConfig;
+import org.apache.usergrid.persistence.core.astyanax.MultiTennantColumnFamily;
+import org.apache.usergrid.persistence.core.astyanax.ScopedRowKey;
+import org.apache.usergrid.persistence.core.consistency.TimeService;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.core.util.ValidationUtils;
+import org.apache.usergrid.persistence.graph.Edge;
+import org.apache.usergrid.persistence.graph.GraphFig;
+import org.apache.usergrid.persistence.graph.MarkedEdge;
+import org.apache.usergrid.persistence.graph.SearchByEdge;
+import org.apache.usergrid.persistence.graph.SearchByEdgeType;
+import org.apache.usergrid.persistence.graph.SearchByIdType;
+import org.apache.usergrid.persistence.graph.impl.SimpleMarkedEdge;
+import org.apache.usergrid.persistence.graph.serialization.impl.shard.DirectedEdge;
+import org.apache.usergrid.persistence.graph.serialization.impl.shard.DirectedEdgeMeta;
+import org.apache.usergrid.persistence.graph.serialization.impl.shard.EdgeColumnFamilies;
+import org.apache.usergrid.persistence.graph.serialization.impl.shard.EdgeRowKey;
+import org.apache.usergrid.persistence.graph.serialization.impl.shard.EdgeShardStrategy;
+import org.apache.usergrid.persistence.graph.serialization.impl.shard.NodeType;
+import org.apache.usergrid.persistence.graph.serialization.impl.shard.RowKey;
+import org.apache.usergrid.persistence.graph.serialization.impl.shard.RowKeyType;
+import org.apache.usergrid.persistence.graph.serialization.impl.shard.Shard;
+import org.apache.usergrid.persistence.graph.serialization.impl.shard.ShardEntryGroup;
+import org.apache.usergrid.persistence.graph.serialization.impl.shard.ShardedEdgeSerialization;
+import org.apache.usergrid.persistence.graph.serialization.util.EdgeUtils;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.fasterxml.uuid.UUIDComparator;
+import com.fasterxml.uuid.impl.UUIDUtil;
+import com.google.common.base.Optional;
+import com.google.inject.Singleton;
+import com.netflix.astyanax.Keyspace;
+import com.netflix.astyanax.MutationBatch;
+import com.netflix.astyanax.Serializer;
+import com.netflix.astyanax.util.RangeBuilder;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+
+@Singleton
+public class ShardedEdgeSerializationImpl implements ShardedEdgeSerialization {
+
+    protected final Keyspace keyspace;
+    protected final CassandraConfig cassandraConfig;
+    protected final GraphFig graphFig;
+    protected final EdgeShardStrategy writeEdgeShardStrategy;
+    protected final TimeService timeService;
+
+
+    @Inject
+    public ShardedEdgeSerializationImpl( final Keyspace keyspace, final CassandraConfig cassandraConfig,
+                                         final GraphFig graphFig, final EdgeShardStrategy writeEdgeShardStrategy,
+                                         final TimeService timeService ) {
+
+
+        checkNotNull( "keyspace required", keyspace );
+        checkNotNull( "cassandraConfig required", cassandraConfig );
+        checkNotNull( "consistencyFig required", graphFig );
+        checkNotNull( "writeEdgeShardStrategy required", writeEdgeShardStrategy );
+        checkNotNull( "timeService required", timeService );
+
+
+        this.keyspace = keyspace;
+        this.cassandraConfig = cassandraConfig;
+        this.graphFig = graphFig;
+        this.writeEdgeShardStrategy = writeEdgeShardStrategy;
+        this.timeService = timeService;
+    }
+
+
+    @Override
+    public MutationBatch writeEdge( final EdgeColumnFamilies columnFamilies, final ApplicationScope scope,
+                                    final MarkedEdge markedEdge, final UUID timestamp  ) {
+        ValidationUtils.validateApplicationScope( scope );
+        EdgeUtils.validateEdge( markedEdge );
+        ValidationUtils.verifyTimeUuid( timestamp, "timestamp" );
+
+
+        final MutationBatch batch = keyspace.prepareMutationBatch().withConsistencyLevel( cassandraConfig.getWriteCL() )
+                                            .withTimestamp( timestamp.timestamp() );
+
+        final boolean isDeleted = markedEdge.isDeleted();
+
+
+        doWrite( columnFamilies, scope, markedEdge, new RowOp<RowKey>() {
+            @Override
+            public void writeEdge( final MultiTennantColumnFamily<ApplicationScope, RowKey, DirectedEdge> columnFamily,
+                                   final RowKey rowKey, final DirectedEdge edge ) {
+                batch.withRow( columnFamily, ScopedRowKey.fromKey( scope, rowKey ) ).putColumn( edge, isDeleted );
+            }
+
+
+            @Override
+            public void countEdge( final Shard shard, final DirectedEdgeMeta directedEdgeMeta ) {
+               if(!isDeleted) {
+                   writeEdgeShardStrategy.increment( scope, shard, 1,  directedEdgeMeta );
+               }
+            }
+
+
+
+            @Override
+            public void writeVersion( final MultiTennantColumnFamily<ApplicationScope, EdgeRowKey, Long> columnFamily,
+                                      final EdgeRowKey rowKey, final long timestamp ) {
+                batch.withRow( columnFamily, ScopedRowKey.fromKey( scope, rowKey ) ).putColumn( timestamp, isDeleted );
+            }
+        } );
+
+
+        return batch;
+    }
+
+
+    @Override
+    public MutationBatch deleteEdge( final EdgeColumnFamilies columnFamilies, final ApplicationScope scope,
+                                     final MarkedEdge markedEdge, final UUID timestamp ) {
+        ValidationUtils.validateApplicationScope( scope );
+        EdgeUtils.validateEdge( markedEdge );
+        ValidationUtils.verifyTimeUuid( timestamp, "timestamp" );
+
+
+        final MutationBatch batch = keyspace.prepareMutationBatch().withConsistencyLevel( cassandraConfig.getWriteCL() )
+                                            .withTimestamp( timestamp.timestamp() );
+
+
+        doWrite( columnFamilies, scope, markedEdge, new RowOp<RowKey>() {
+            @Override
+            public void writeEdge( final MultiTennantColumnFamily<ApplicationScope, RowKey, DirectedEdge> columnFamily,
+                                   final RowKey rowKey, final DirectedEdge edge ) {
+                batch.withRow( columnFamily, ScopedRowKey.fromKey( scope, rowKey ) ).deleteColumn( edge );
+            }
+
+
+            @Override
+            public void countEdge(  final Shard shard, final DirectedEdgeMeta directedEdgeMeta ) {
+                writeEdgeShardStrategy.increment( scope,  shard, -1, directedEdgeMeta );
+            }
+
+
+            @Override
+            public void writeVersion( final MultiTennantColumnFamily<ApplicationScope, EdgeRowKey, Long> columnFamily,
+                                      final EdgeRowKey rowKey, final long timestamp ) {
+                batch.withRow( columnFamily, ScopedRowKey.fromKey( scope, rowKey ) ).deleteColumn( timestamp );
+            }
+        } );
+
+
+        return batch;
+    }
+
+
+    /**
+     * EdgeWrite the edges internally
+     *
+     * @param scope The scope to encapsulate
+     * @param edge The edge to write
+     * @param op The row operation to invoke
+     */
+    private void doWrite( final EdgeColumnFamilies columnFamilies, final ApplicationScope scope, final MarkedEdge edge,
+                          final RowOp op ) {
+        ValidationUtils.validateApplicationScope( scope );
+        EdgeUtils.validateEdge( edge );
+
+        final Id sourceNodeId = edge.getSourceNode();
+        final String sourceNodeType = sourceNodeId.getType();
+        final Id targetNodeId = edge.getTargetNode();
+        final String targetNodeType = targetNodeId.getType();
+        final long timestamp = edge.getTimestamp();
+        final String type = edge.getType();
+
+
+        /**
+         * Key in the serializers based on the edge
+         */
+
+
+        /**
+         * write edges from source->target
+         */
+
+
+        final long time = timeService.getCurrentTime();
+
+        final DirectedEdge sourceEdge = new DirectedEdge( targetNodeId, timestamp );
+
+        final DirectedEdgeMeta sourceEdgeMeta =  DirectedEdgeMeta.fromSourceNode( sourceNodeId, type );
+
+
+
+        final ShardEntryGroup sourceRowKeyShard =
+                writeEdgeShardStrategy.getWriteShards( scope, timestamp, sourceEdgeMeta );
+
+        final MultiTennantColumnFamily<ApplicationScope, RowKey, DirectedEdge> sourceCf =
+                columnFamilies.getSourceNodeCfName();
+
+
+        for ( Shard shard : sourceRowKeyShard.getWriteShards(time) ) {
+
+            final long shardId = shard.getShardIndex();
+            final RowKey sourceRowKey = new RowKey( sourceNodeId, type, shardId );
+            op.writeEdge( sourceCf, sourceRowKey, sourceEdge );
+            op.countEdge( shard, sourceEdgeMeta );
+        }
+
+
+
+        final DirectedEdgeMeta sourceEdgeTargetTypeMeta =  DirectedEdgeMeta.fromSourceNodeTargetType( sourceNodeId, type, targetNodeType );
+
+
+        final ShardEntryGroup sourceWithTypeRowKeyShard = writeEdgeShardStrategy
+                .getWriteShards( scope, timestamp, sourceEdgeTargetTypeMeta );
+
+        final MultiTennantColumnFamily<ApplicationScope, RowKeyType, DirectedEdge> targetCf =
+                columnFamilies.getSourceNodeTargetTypeCfName();
+
+        for ( Shard shard : sourceWithTypeRowKeyShard.getWriteShards(time) ) {
+
+            final long shardId = shard.getShardIndex();
+            final RowKeyType sourceRowKeyType = new RowKeyType( sourceNodeId, type, targetNodeId, shardId );
+
+            op.writeEdge( targetCf, sourceRowKeyType, sourceEdge );
+            op.countEdge( shard, sourceEdgeTargetTypeMeta );
+        }
+
+
+        /**
+         * write edges from target<-source
+         */
+
+        final DirectedEdge targetEdge = new DirectedEdge( sourceNodeId, timestamp );
+
+        final DirectedEdgeMeta targetEdgeMeta =  DirectedEdgeMeta.fromTargetNode( targetNodeId, type );
+
+
+
+        final ShardEntryGroup targetRowKeyShard =
+                writeEdgeShardStrategy.getWriteShards( scope, timestamp, targetEdgeMeta );
+
+        final MultiTennantColumnFamily<ApplicationScope, RowKey, DirectedEdge> sourceByTargetCf =
+                columnFamilies.getTargetNodeCfName();
+
+        for ( Shard shard : targetRowKeyShard.getWriteShards(time) ) {
+            final long shardId = shard.getShardIndex();
+            final RowKey targetRowKey = new RowKey( targetNodeId, type, shardId );
+
+            op.writeEdge( sourceByTargetCf, targetRowKey, targetEdge );
+            op.countEdge( shard, targetEdgeMeta );
+        }
+
+
+
+        final DirectedEdgeMeta targetEdgeSourceTypeMeta =  DirectedEdgeMeta.fromTargetNodeSourceType( targetNodeId, type, sourceNodeType );
+
+
+        final ShardEntryGroup targetWithTypeRowKeyShard = writeEdgeShardStrategy
+                .getWriteShards( scope, timestamp, targetEdgeSourceTypeMeta );
+
+        final MultiTennantColumnFamily<ApplicationScope, RowKeyType, DirectedEdge> targetBySourceCf =
+                columnFamilies.getTargetNodeSourceTypeCfName();
+
+
+        for ( Shard shard : targetWithTypeRowKeyShard.getWriteShards(time) ) {
+
+            final long shardId = shard.getShardIndex();
+
+            final RowKeyType targetRowKeyType = new RowKeyType( targetNodeId, type, sourceNodeId, shardId );
+
+
+            op.writeEdge( targetBySourceCf, targetRowKeyType, targetEdge );
+            op.countEdge( shard, targetEdgeSourceTypeMeta );
+        }
+
+        /**
+         * Always a 0l shard, we're hard limiting 2b timestamps for the same edge
+         */
+        final EdgeRowKey edgeRowKey = new EdgeRowKey( sourceNodeId, type, targetNodeId, 0l );
+
+
+        /**
+         * Write this in the timestamp log for this edge of source->target
+         */
+        op.writeVersion( columnFamilies.getGraphEdgeVersions(), edgeRowKey, timestamp );
+    }
+
+
+    @Override
+    public Iterator<MarkedEdge> getEdgeVersions( final EdgeColumnFamilies columnFamilies, final ApplicationScope scope,
+                                                 final SearchByEdge search, final Iterator<ShardEntryGroup> shards ) {
+        ValidationUtils.validateApplicationScope( scope );
+        EdgeUtils.validateSearchByEdge( search );
+
+        final Id targetId = search.targetNode();
+        final Id sourceId = search.sourceNode();
+        final String type = search.getType();
+        final long maxTimestamp = search.getMaxTimestamp();
+        final MultiTennantColumnFamily<ApplicationScope, EdgeRowKey, Long> columnFamily =
+                columnFamilies.getGraphEdgeVersions();
+        final Serializer<Long> serializer = columnFamily.getColumnSerializer();
+
+        final EdgeSearcher<EdgeRowKey, Long, MarkedEdge> searcher =
+                new EdgeSearcher<EdgeRowKey, Long, MarkedEdge>( scope, maxTimestamp, search.last(), shards ) {
+
+                    @Override
+                    protected Serializer<Long> getSerializer() {
+                        return serializer;
+                    }
+
+
+                    @Override
+                    public void setRange( final RangeBuilder builder ) {
+
+
+                        if ( last.isPresent() ) {
+                            super.setRange( builder );
+                            return;
+                        }
+
+                        //start seeking at a value < our max version
+                        builder.setStart( maxTimestamp );
+                    }
+
+
+                    @Override
+                    protected EdgeRowKey generateRowKey( long shard ) {
+                        return new EdgeRowKey( sourceId, type, targetId, shard );
+                    }
+
+
+                    @Override
+                    protected Long getStartColumn( final Edge last ) {
+                        return last.getTimestamp();
+                    }
+
+
+                    @Override
+                    protected MarkedEdge createEdge( final Long column, final boolean marked ) {
+                        return new SimpleMarkedEdge( sourceId, type, targetId, column.longValue(), marked );
+                    }
+
+
+                    @Override
+                    public int compare( final MarkedEdge o1, final MarkedEdge o2 ) {
+                        return Long.compare( o1.getTimestamp(), o2.getTimestamp() );
+                    }
+                };
+
+        return new ShardRowIterator<>( searcher, columnFamily, keyspace, cassandraConfig.getReadCL(),
+                graphFig.getScanPageSize() );
+    }
+
+
+    @Override
+    public Iterator<MarkedEdge> getEdgesFromSource( final EdgeColumnFamilies columnFamilies,
+                                                    final ApplicationScope scope, final SearchByEdgeType edgeType,
+                                                    final Iterator<ShardEntryGroup> shards ) {
+
+        ValidationUtils.validateApplicationScope( scope );
+        EdgeUtils.validateSearchByEdgeType( edgeType );
+
+        final Id sourceId = edgeType.getNode();
+        final String type = edgeType.getType();
+        final long maxTimestamp = edgeType.getMaxTimestamp();
+        final MultiTennantColumnFamily<ApplicationScope, RowKey, DirectedEdge> columnFamily =
+                columnFamilies.getSourceNodeCfName();
+        final Serializer<DirectedEdge> serializer = columnFamily.getColumnSerializer();
+
+
+        final SourceEdgeSearcher<RowKey, DirectedEdge, MarkedEdge> searcher =
+                new SourceEdgeSearcher<RowKey, DirectedEdge, MarkedEdge>( scope, maxTimestamp, edgeType.last(), shards ) {
+
+
+                    @Override
+                    protected Serializer<DirectedEdge> getSerializer() {
+                        return serializer;
+                    }
+
+
+                    @Override
+                    protected RowKey generateRowKey( long shard ) {
+                        return new RowKey( sourceId, type, shard );
+                    }
+
+
+                    @Override
+                    protected DirectedEdge getStartColumn( final Edge last ) {
+                        return new DirectedEdge( last.getTargetNode(), last.getTimestamp() );
+                    }
+
+
+                    @Override
+                    protected MarkedEdge createEdge( final DirectedEdge edge, final boolean marked ) {
+                        return new SimpleMarkedEdge( sourceId, type, edge.id, edge.timestamp, marked );
+                    }
+
+
+                };
+
+
+        return new ShardRowIterator<>( searcher, columnFamily, keyspace, cassandraConfig.getReadCL(),
+                graphFig.getScanPageSize() );
+    }
+
+
+    @Override
+    public Iterator<MarkedEdge> getEdgesFromSourceByTargetType( final EdgeColumnFamilies columnFamilies,
+                                                                final ApplicationScope scope,
+                                                                final SearchByIdType edgeType,
+                                                                final Iterator<ShardEntryGroup> shards ) {
+
+        ValidationUtils.validateApplicationScope( scope );
+        EdgeUtils.validateSearchByEdgeType( edgeType );
+
+        final Id targetId = edgeType.getNode();
+        final String type = edgeType.getType();
+        final String targetType = edgeType.getIdType();
+        final long maxTimestamp = edgeType.getMaxTimestamp();
+        final MultiTennantColumnFamily<ApplicationScope, RowKeyType, DirectedEdge> columnFamily =
+                columnFamilies.getSourceNodeTargetTypeCfName();
+        final Serializer<DirectedEdge> serializer = columnFamily.getColumnSerializer();
+
+
+        final SourceEdgeSearcher<RowKeyType, DirectedEdge, MarkedEdge> searcher =
+                new SourceEdgeSearcher<RowKeyType, DirectedEdge, MarkedEdge>( scope, maxTimestamp, edgeType.last(), shards ) {
+
+                    @Override
+                    protected Serializer<DirectedEdge> getSerializer() {
+                        return serializer;
+                    }
+
+
+                    @Override
+                    protected RowKeyType generateRowKey( long shard ) {
+                        return new RowKeyType( targetId, type, targetType, shard );
+                    }
+
+
+                    @Override
+                    protected DirectedEdge getStartColumn( final Edge last ) {
+                        return new DirectedEdge( last.getTargetNode(), last.getTimestamp() );
+                    }
+
+
+                    @Override
+                    protected MarkedEdge createEdge( final DirectedEdge edge, final boolean marked ) {
+                        return new SimpleMarkedEdge( targetId, type, edge.id, edge.timestamp, marked );
+                    }
+
+
+
+                };
+
+        return new ShardRowIterator( searcher, columnFamily, keyspace, cassandraConfig.getReadCL(),
+                graphFig.getScanPageSize() );
+    }
+
+
+
+
+    @Override
+    public Iterator<MarkedEdge> getEdgesToTarget( final EdgeColumnFamilies columnFamilies, final ApplicationScope scope,
+                                                  final SearchByEdgeType edgeType,
+                                                  final Iterator<ShardEntryGroup> shards ) {
+        ValidationUtils.validateApplicationScope( scope );
+        EdgeUtils.validateSearchByEdgeType( edgeType );
+
+        final Id targetId = edgeType.getNode();
+        final String type = edgeType.getType();
+        final long maxTimestamp = edgeType.getMaxTimestamp();
+        final MultiTennantColumnFamily<ApplicationScope, RowKey, DirectedEdge> columnFamily =
+                columnFamilies.getTargetNodeCfName();
+        final Serializer<DirectedEdge> serializer = columnFamily.getColumnSerializer();
+
+        final TargetEdgeSearcher<RowKey, DirectedEdge, MarkedEdge> searcher =
+                new TargetEdgeSearcher<RowKey, DirectedEdge, MarkedEdge>( scope, maxTimestamp, edgeType.last(), shards ) {
+
+                    @Override
+                    protected Serializer<DirectedEdge> getSerializer() {
+                        return serializer;
+                    }
+
+
+                    @Override
+                    protected RowKey generateRowKey( long shard ) {
+                        return new RowKey( targetId, type, shard );
+                    }
+
+
+                    @Override
+                    protected DirectedEdge getStartColumn( final Edge last ) {
+                        return new DirectedEdge( last.getSourceNode(), last.getTimestamp() );
+                    }
+
+
+                    @Override
+                    protected MarkedEdge createEdge( final DirectedEdge edge, final boolean marked ) {
+                        return new SimpleMarkedEdge( edge.id, type, targetId, edge.timestamp, marked );
+                    }
+                };
+
+
+        return new ShardRowIterator<>( searcher, columnFamily, keyspace, cassandraConfig.getReadCL(),
+                graphFig.getScanPageSize() );
+    }
+
+
+    @Override
+    public Iterator<MarkedEdge> getEdgesToTargetBySourceType( final EdgeColumnFamilies columnFamilies,
+                                                              final ApplicationScope scope,
+                                                              final SearchByIdType edgeType,
+                                                              final Iterator<ShardEntryGroup> shards ) {
+
+        ValidationUtils.validateApplicationScope( scope );
+        EdgeUtils.validateSearchByEdgeType( edgeType );
+
+        final Id targetId = edgeType.getNode();
+        final String sourceType = edgeType.getIdType();
+        final String type = edgeType.getType();
+        final long maxTimestamp = edgeType.getMaxTimestamp();
+        final MultiTennantColumnFamily<ApplicationScope, RowKeyType, DirectedEdge> columnFamily =
+                columnFamilies.getTargetNodeSourceTypeCfName();
+        final Serializer<DirectedEdge> serializer = columnFamily.getColumnSerializer();
+
+
+        final TargetEdgeSearcher<RowKeyType, DirectedEdge, MarkedEdge> searcher =
+                new TargetEdgeSearcher<RowKeyType, DirectedEdge, MarkedEdge>( scope, maxTimestamp, edgeType.last(), shards ) {
+                    @Override
+                    protected Serializer<DirectedEdge> getSerializer() {
+                        return serializer;
+                    }
+
+
+                    @Override
+                    protected RowKeyType generateRowKey( final long shard ) {
+                        return new RowKeyType( targetId, type, sourceType, shard );
+                    }
+
+
+                    @Override
+                    protected DirectedEdge getStartColumn( final Edge last ) {
+                        return new DirectedEdge( last.getTargetNode(), last.getTimestamp() );
+                    }
+
+
+                    @Override
+                    protected MarkedEdge createEdge( final DirectedEdge edge, final boolean marked ) {
+                        return new SimpleMarkedEdge( edge.id, type, targetId, edge.timestamp, marked );
+                    }
+                };
+
+        return new ShardRowIterator<>( searcher, columnFamily, keyspace, cassandraConfig.getReadCL(),
+                graphFig.getScanPageSize() );
+    }
+
+
+    /**
+     * Class for performing searched on rows based on source id
+     */
+    private static abstract class SourceEdgeSearcher<R, C, T extends Edge> extends EdgeSearcher<R, C, T>{
+
+        protected SourceEdgeSearcher( final ApplicationScope scope, final long maxTimestamp, final Optional<Edge> last,
+                                      final Iterator<ShardEntryGroup> shards ) {
+            super( scope, maxTimestamp, last, shards );
+        }
+
+
+        public int compare( final T o1, final T o2 ) {
+            int compare = Long.compare(o1.getTimestamp(), o2.getTimestamp());
+
+            if(compare == 0){
+                compare = o1.getTargetNode().compareTo( o2.getTargetNode());
+            }
+
+            return compare;
+        }
+
+
+    }
+
+
+    /**
+     * Class for performing searched on rows based on target id
+     */
+    private static abstract class TargetEdgeSearcher<R, C, T extends Edge> extends EdgeSearcher<R, C, T>{
+
+        protected TargetEdgeSearcher( final ApplicationScope scope, final long maxTimestamp, final Optional<Edge> last,
+                                      final Iterator<ShardEntryGroup> shards ) {
+            super( scope, maxTimestamp, last, shards );
+        }
+
+
+        public int compare( final T o1, final T o2 ) {
+            int compare = Long.compare(o1.getTimestamp(), o2.getTimestamp());
+
+            if(compare == 0){
+                compare = o1.getTargetNode().compareTo( o2.getTargetNode());
+            }
+
+            return compare;
+        }
+
+
+    }
+
+    /**
+     * Simple callback to perform puts and deletes with a common row setup code
+     *
+     * @param <R> The row key type
+     */
+    private static interface RowOp<R> {
+
+        /**
+         * Write the edge with the given data
+         */
+        void writeEdge( final MultiTennantColumnFamily<ApplicationScope, R, DirectedEdge> columnFamily, final  R rowKey,
+                        final DirectedEdge edge );
+
+        /**
+         * Perform the count on the edge
+         */
+        void countEdge( final Shard shard, final DirectedEdgeMeta directedEdgeMeta);
+
+        /**
+         * Write the edge into the version cf
+         */
+        void writeVersion( final MultiTennantColumnFamily<ApplicationScope, EdgeRowKey, Long> columnFamily,
+                           final EdgeRowKey rowKey, long timestamp );
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e9d652dd/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/SizebasedEdgeColumnFamilies.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/SizebasedEdgeColumnFamilies.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/SizebasedEdgeColumnFamilies.java
new file mode 100644
index 0000000..9050b0a
--- /dev/null
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/SizebasedEdgeColumnFamilies.java
@@ -0,0 +1,150 @@
+/*
+ *
+ *  * Licensed to the Apache Software Foundation (ASF) under one
+ *  * or more contributor license agreements.  See the NOTICE file
+ *  * distributed with this work for additional information
+ *  * regarding copyright ownership.  The ASF licenses this file
+ *  * to you under the Apache License, Version 2.0 (the
+ *  * "License"); you may not use this file except in compliance
+ *  * with the License.  You may obtain a copy of the License at
+ *  *
+ *  *    http://www.apache.org/licenses/LICENSE-2.0
+ *  *
+ *  * Unless required by applicable law or agreed to in writing,
+ *  * software distributed under the License is distributed on an
+ *  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ *  * KIND, either express or implied.  See the License for the
+ *  * specific language governing permissions and limitations
+ *  * under the License.
+ *
+ */
+package org.apache.usergrid.persistence.graph.serialization.impl.shard.impl;
+
+
+import java.util.Arrays;
+import java.util.Collection;
+
+import org.apache.cassandra.db.marshal.BytesType;
+import org.apache.cassandra.db.marshal.DynamicCompositeType;
+
+import org.apache.usergrid.persistence.core.astyanax.ColumnTypes;
+import org.apache.usergrid.persistence.core.astyanax.MultiTennantColumnFamily;
+import org.apache.usergrid.persistence.core.astyanax.MultiTennantColumnFamilyDefinition;
+import org.apache.usergrid.persistence.core.astyanax.OrganizationScopedRowKeySerializer;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.graph.serialization.impl.shard.DirectedEdge;
+import org.apache.usergrid.persistence.graph.serialization.impl.shard.EdgeColumnFamilies;
+import org.apache.usergrid.persistence.graph.serialization.impl.shard.EdgeRowKey;
+import org.apache.usergrid.persistence.graph.serialization.impl.shard.RowKey;
+import org.apache.usergrid.persistence.graph.serialization.impl.shard.RowKeyType;
+
+import com.netflix.astyanax.serializers.LongSerializer;
+
+import static org.apache.usergrid.persistence.core.astyanax.ColumnTypes.LONG_TYPE_REVERSED;
+import static org.apache.usergrid.persistence.core.astyanax.ColumnTypes.UUID_TYPE_REVERSED;
+
+
+/**
+ * Implementation of size based column family
+ */
+public class SizebasedEdgeColumnFamilies implements EdgeColumnFamilies {
+
+
+    //Row key with no type
+    private static final RowSerializer ROW_SERIALIZER = new RowSerializer();
+
+    //row key with target id type
+    private static final RowTypeSerializer ROW_TYPE_SERIALIZER = new RowTypeSerializer();
+
+    private static final EdgeRowKeySerializer EDGE_ROW_KEY_SERIALIZER = new EdgeRowKeySerializer();
+
+    //Edge serializers
+    private static final EdgeSerializer EDGE_SERIALIZER = new EdgeSerializer();
+
+    private static final LongSerializer LONG_SERIALIZER = LongSerializer.get();
+
+    private static final String EDGE_DYNAMIC_COMPOSITE_TYPE =
+            //we purposefully associate lower case "l" and "u" with reversed types.  This way we can use
+            //the default serialization in Astayanax, but get reverse order in cassandra
+            DynamicCompositeType.class.getSimpleName() + "(s=>UTF8Type,l=>" + LONG_TYPE_REVERSED + ",u=>"
+                    + UUID_TYPE_REVERSED + ")";
+
+
+    //initialize the CF's from our implementation
+    private static final MultiTennantColumnFamily<ApplicationScope, RowKey, DirectedEdge> SOURCE_NODE_EDGES =
+            new MultiTennantColumnFamily<>( "Graph_Source_Node_Edges",
+                    new OrganizationScopedRowKeySerializer<>( ROW_SERIALIZER ), EDGE_SERIALIZER );
+
+
+    private static final MultiTennantColumnFamily<ApplicationScope, RowKey, DirectedEdge> TARGET_NODE_EDGES =
+            new MultiTennantColumnFamily<>( "Graph_Target_Node_Edges",
+                    new OrganizationScopedRowKeySerializer<>( ROW_SERIALIZER ), EDGE_SERIALIZER );
+
+
+    private static final MultiTennantColumnFamily<ApplicationScope, RowKeyType, DirectedEdge> SOURCE_NODE_TARGET_TYPE =
+            new MultiTennantColumnFamily<>( "Graph_Source_Node_Target_Type",
+                    new OrganizationScopedRowKeySerializer<>( ROW_TYPE_SERIALIZER ), EDGE_SERIALIZER );
+
+
+    /**
+     * The edges that are to the target node with the source type.  The target node is the row key
+     */
+    private static final MultiTennantColumnFamily<ApplicationScope, RowKeyType, DirectedEdge> TARGET_NODE_SOURCE_TYPE =
+            new MultiTennantColumnFamily<>( "Graph_Target_Node_Source_Type",
+                    new OrganizationScopedRowKeySerializer<>( ROW_TYPE_SERIALIZER ), EDGE_SERIALIZER );
+
+
+    private static final MultiTennantColumnFamily<ApplicationScope, EdgeRowKey, Long> EDGE_VERSIONS =
+            new MultiTennantColumnFamily<>( "Graph_Edge_Versions",
+                    new OrganizationScopedRowKeySerializer<>( EDGE_ROW_KEY_SERIALIZER ), LONG_SERIALIZER );
+
+
+    @Override
+    public MultiTennantColumnFamily<ApplicationScope, RowKey, DirectedEdge> getSourceNodeCfName() {
+        return SOURCE_NODE_EDGES;
+    }
+
+
+    @Override
+    public MultiTennantColumnFamily<ApplicationScope, RowKey, DirectedEdge> getTargetNodeCfName() {
+        return TARGET_NODE_EDGES;
+    }
+
+
+    @Override
+    public MultiTennantColumnFamily<ApplicationScope, RowKeyType, DirectedEdge> getSourceNodeTargetTypeCfName() {
+        return SOURCE_NODE_TARGET_TYPE;
+    }
+
+
+    @Override
+    public MultiTennantColumnFamily<ApplicationScope, RowKeyType, DirectedEdge> getTargetNodeSourceTypeCfName() {
+        return TARGET_NODE_SOURCE_TYPE;
+    }
+
+
+    @Override
+    public MultiTennantColumnFamily<ApplicationScope, EdgeRowKey, Long> getGraphEdgeVersions() {
+        return EDGE_VERSIONS;
+    }
+
+
+    @Override
+    public Collection<MultiTennantColumnFamilyDefinition> getColumnFamilies() {
+        return Arrays
+                .asList( graphCf( SOURCE_NODE_EDGES ), graphCf( TARGET_NODE_EDGES ), graphCf( SOURCE_NODE_TARGET_TYPE ),
+                        graphCf( TARGET_NODE_SOURCE_TYPE ),
+                        new MultiTennantColumnFamilyDefinition( EDGE_VERSIONS, BytesType.class.getSimpleName(),
+                                ColumnTypes.LONG_TYPE_REVERSED, BytesType.class.getSimpleName(),
+                                MultiTennantColumnFamilyDefinition.CacheOption.KEYS ) );
+    }
+
+
+    /**
+     * Helper to generate an edge definition by the type
+     */
+    private MultiTennantColumnFamilyDefinition graphCf( MultiTennantColumnFamily cf ) {
+        return new MultiTennantColumnFamilyDefinition( cf, BytesType.class.getSimpleName(), EDGE_DYNAMIC_COMPOSITE_TYPE,
+                BytesType.class.getSimpleName(), MultiTennantColumnFamilyDefinition.CacheOption.KEYS );
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e9d652dd/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/SizebasedEdgeShardStrategy.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/SizebasedEdgeShardStrategy.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/SizebasedEdgeShardStrategy.java
index f246f23..6558273 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/SizebasedEdgeShardStrategy.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/SizebasedEdgeShardStrategy.java
@@ -23,9 +23,14 @@ package org.apache.usergrid.persistence.graph.serialization.impl.shard.impl;
 import java.util.Iterator;
 
 import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.graph.serialization.impl.shard.DirectedEdgeMeta;
 import org.apache.usergrid.persistence.graph.serialization.impl.shard.EdgeShardStrategy;
 import org.apache.usergrid.persistence.graph.serialization.impl.shard.NodeShardApproximation;
 import org.apache.usergrid.persistence.graph.serialization.impl.shard.NodeShardCache;
+import org.apache.usergrid.persistence.graph.serialization.impl.shard.NodeType;
+import org.apache.usergrid.persistence.graph.serialization.impl.shard.Shard;
+import org.apache.usergrid.persistence.graph.serialization.impl.shard.ShardEntryGroup;
+import org.apache.usergrid.persistence.graph.serialization.impl.shard.ShardOperator;
 import org.apache.usergrid.persistence.model.entity.Id;
 
 import com.google.inject.Inject;
@@ -52,52 +57,21 @@ public class SizebasedEdgeShardStrategy implements EdgeShardStrategy {
 
 
     @Override
-    public long getWriteShard( final ApplicationScope scope, final Id rowKeyId, final long timestamp,
-                               final String... types ) {
-        return shardCache.getSlice( scope, rowKeyId, timestamp, types );
+    public ShardEntryGroup getWriteShards( final ApplicationScope scope,
+                                        final long timestamp, final DirectedEdgeMeta directedEdgeMeta ) {
+        return shardCache.getWriteShardGroup( scope, timestamp, directedEdgeMeta);
     }
 
 
     @Override
-    public Iterator<Long> getReadShards( final ApplicationScope scope, final Id rowKeyId, final long maxTimestamp,
-                                         final String... types ) {
-        return shardCache.getVersions( scope, rowKeyId, maxTimestamp, types );
+    public Iterator<ShardEntryGroup> getReadShards( final ApplicationScope scope, final long maxTimestamp, final DirectedEdgeMeta directedEdgeMeta ) {
+        return shardCache.getReadShardGroup( scope, maxTimestamp, directedEdgeMeta );
     }
 
 
     @Override
-    public void increment( final ApplicationScope scope, final Id rowKeyId, final long shardId, final long count,
-                           final String... types ) {
-        shardApproximation.increment( scope, rowKeyId, shardId, count, types );
-    }
-
-
-    @Override
-    public String getSourceNodeCfName() {
-        return "Graph_Source_Node_Edges";
-    }
-
-
-    @Override
-    public String getTargetNodeCfName() {
-        return "Graph_Target_Node_Edges";
-    }
-
-
-    @Override
-    public String getSourceNodeTargetTypeCfName() {
-        return "Graph_Source_Node_Target_Type";
-    }
-
-
-    @Override
-    public String getTargetNodeSourceTypeCfName() {
-        return "Graph_Target_Node_Source_Type";
-    }
-
-
-    @Override
-    public String getGraphEdgeVersions() {
-        return "Graph_Edge_Versions";
+    public void increment( final ApplicationScope scope, final Shard shard,
+                           final long count, final DirectedEdgeMeta directedEdgeMeta) {
+        shardApproximation.increment( scope, shard,  count, directedEdgeMeta );
     }
 }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e9d652dd/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/SourceEdgeSearcher.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/SourceEdgeSearcher.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/SourceEdgeSearcher.java
new file mode 100644
index 0000000..b33fcaf
--- /dev/null
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/SourceEdgeSearcher.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.usergrid.persistence.graph.serialization.impl.shard.impl;
+
+
+public class SourceEdgeSearcher {}

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e9d652dd/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/ComittedGraphManagerIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/ComittedGraphManagerIT.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/ComittedGraphManagerIT.java
deleted file mode 100644
index 596480d..0000000
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/ComittedGraphManagerIT.java
+++ /dev/null
@@ -1,138 +0,0 @@
-/*
- *
- *  * Licensed to the Apache Software Foundation (ASF) under one
- *  * or more contributor license agreements.  See the NOTICE file
- *  * distributed with this work for additional information
- *  * regarding copyright ownership.  The ASF licenses this file
- *  * to you under the Apache License, Version 2.0 (the
- *  * "License"); you may not use this file except in compliance
- *  * with the License.  You may obtain a copy of the License at
- *  *
- *  *    http://www.apache.org/licenses/LICENSE-2.0
- *  *
- *  * Unless required by applicable law or agreed to in writing,
- *  * software distributed under the License is distributed on an
- *  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- *  * KIND, either express or implied.  See the License for the
- *  * specific language governing permissions and limitations
- *  * under the License.
- *
- */
-package org.apache.usergrid.persistence.graph;
-
-
-import java.util.concurrent.TimeUnit;
-
-import org.jukito.JukitoRunner;
-import org.jukito.UseModules;
-import org.junit.runner.RunWith;
-
-import org.apache.usergrid.persistence.core.cassandra.ITRunner;
-import org.apache.usergrid.persistence.graph.guice.TestGraphModule;
-import org.apache.usergrid.persistence.model.entity.Id;
-
-import rx.Observable;
-
-
-/**
- * Integration test that performs all calls immediately after writes without blocking.  Tests that our
- * view is immediately consistent to our users, even if we have yet to perform background processing
- */
-@RunWith(ITRunner.class)
-@UseModules({ TestGraphModule.class })
-public class ComittedGraphManagerIT extends GraphManagerIT {
-
-
-    @Override
-    protected GraphManager getHelper(GraphManager gm) {
-        return new ComittedGraphTestHelper( gm );
-    }
-
-
-    /**
-     * Doesn't wait for the async process to happen before returning.  Simply executes and immediately returns.
-     */
-    public static class ComittedGraphTestHelper implements GraphManager {
-
-        private final GraphManager graphManager;
-
-
-        public ComittedGraphTestHelper( final GraphManager graphManager ) {
-            this.graphManager = graphManager;
-        }
-
-
-        @Override
-        public Observable<Edge> writeEdge( final Edge edge ) {
-            return graphManager.writeEdge( edge );
-        }
-
-
-        @Override
-        public Observable<Edge> deleteEdge( final Edge edge ) {
-            return graphManager.deleteEdge( edge );
-        }
-
-
-        @Override
-        public Observable<Id> deleteNode( final Id node, final long timestamp) {
-            return graphManager.deleteNode( node, timestamp );
-        }
-
-
-        @Override
-        public Observable<Edge> loadEdgeVersions( final SearchByEdge edge ) {
-            return graphManager.loadEdgeVersions( edge );
-        }
-
-
-        @Override
-        public Observable<Edge> loadEdgesFromSource( final SearchByEdgeType search ) {
-            return graphManager.loadEdgesFromSource( search );
-        }
-
-
-        @Override
-        public Observable<Edge> loadEdgesToTarget( final SearchByEdgeType search ) {
-            return graphManager.loadEdgesToTarget( search );
-        }
-
-
-        @Override
-        public Observable<Edge> loadEdgesFromSourceByType( final SearchByIdType search ) {
-            return  graphManager.loadEdgesFromSourceByType(search);
-        }
-
-
-        @Override
-        public Observable<Edge> loadEdgesToTargetByType( final SearchByIdType search ) {
-            return graphManager.loadEdgesToTargetByType( search );
-        }
-
-
-        @Override
-        public Observable<String> getEdgeTypesFromSource( final SearchEdgeType search ) {
-            return graphManager.getEdgeTypesFromSource( search );
-        }
-
-
-        @Override
-        public Observable<String> getIdTypesFromSource( final SearchIdType search ) {
-            return graphManager.getIdTypesFromSource( search );
-        }
-
-
-        @Override
-        public Observable<String> getEdgeTypesToTarget( final SearchEdgeType search ) {
-            return graphManager.getEdgeTypesToTarget( search );
-        }
-
-
-        @Override
-        public Observable<String> getIdTypesToTarget( final SearchIdType search ) {
-            return graphManager.getIdTypesToTarget( search );
-        }
-
-
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e9d652dd/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/CommittedGraphManagerIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/CommittedGraphManagerIT.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/CommittedGraphManagerIT.java
new file mode 100644
index 0000000..3c04ab4
--- /dev/null
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/CommittedGraphManagerIT.java
@@ -0,0 +1,138 @@
+/*
+ *
+ *  * Licensed to the Apache Software Foundation (ASF) under one
+ *  * or more contributor license agreements.  See the NOTICE file
+ *  * distributed with this work for additional information
+ *  * regarding copyright ownership.  The ASF licenses this file
+ *  * to you under the Apache License, Version 2.0 (the
+ *  * "License"); you may not use this file except in compliance
+ *  * with the License.  You may obtain a copy of the License at
+ *  *
+ *  *    http://www.apache.org/licenses/LICENSE-2.0
+ *  *
+ *  * Unless required by applicable law or agreed to in writing,
+ *  * software distributed under the License is distributed on an
+ *  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ *  * KIND, either express or implied.  See the License for the
+ *  * specific language governing permissions and limitations
+ *  * under the License.
+ *
+ */
+package org.apache.usergrid.persistence.graph;
+
+
+import java.util.concurrent.TimeUnit;
+
+import org.jukito.JukitoRunner;
+import org.jukito.UseModules;
+import org.junit.runner.RunWith;
+
+import org.apache.usergrid.persistence.core.cassandra.ITRunner;
+import org.apache.usergrid.persistence.graph.guice.TestGraphModule;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import rx.Observable;
+
+
+/**
+ * Integration test that performs all calls immediately after writes without blocking.  Tests that our
+ * view is immediately consistent to our users, even if we have yet to perform background processing
+ */
+@RunWith(ITRunner.class)
+@UseModules({ TestGraphModule.class })
+public class CommittedGraphManagerIT extends GraphManagerIT {
+
+
+    @Override
+    protected GraphManager getHelper(GraphManager gm) {
+        return new ComittedGraphTestHelper( gm );
+    }
+
+
+    /**
+     * Doesn't wait for the async process to happen before returning.  Simply executes and immediately returns.
+     */
+    public static class ComittedGraphTestHelper implements GraphManager {
+
+        private final GraphManager graphManager;
+
+
+        public ComittedGraphTestHelper( final GraphManager graphManager ) {
+            this.graphManager = graphManager;
+        }
+
+
+        @Override
+        public Observable<Edge> writeEdge( final Edge edge ) {
+            return graphManager.writeEdge( edge );
+        }
+
+
+        @Override
+        public Observable<Edge> deleteEdge( final Edge edge ) {
+            return graphManager.deleteEdge( edge );
+        }
+
+
+        @Override
+        public Observable<Id> deleteNode( final Id node, final long timestamp) {
+            return graphManager.deleteNode( node, timestamp );
+        }
+
+
+        @Override
+        public Observable<Edge> loadEdgeVersions( final SearchByEdge edge ) {
+            return graphManager.loadEdgeVersions( edge );
+        }
+
+
+        @Override
+        public Observable<Edge> loadEdgesFromSource( final SearchByEdgeType search ) {
+            return graphManager.loadEdgesFromSource( search );
+        }
+
+
+        @Override
+        public Observable<Edge> loadEdgesToTarget( final SearchByEdgeType search ) {
+            return graphManager.loadEdgesToTarget( search );
+        }
+
+
+        @Override
+        public Observable<Edge> loadEdgesFromSourceByType( final SearchByIdType search ) {
+            return  graphManager.loadEdgesFromSourceByType(search);
+        }
+
+
+        @Override
+        public Observable<Edge> loadEdgesToTargetByType( final SearchByIdType search ) {
+            return graphManager.loadEdgesToTargetByType( search );
+        }
+
+
+        @Override
+        public Observable<String> getEdgeTypesFromSource( final SearchEdgeType search ) {
+            return graphManager.getEdgeTypesFromSource( search );
+        }
+
+
+        @Override
+        public Observable<String> getIdTypesFromSource( final SearchIdType search ) {
+            return graphManager.getIdTypesFromSource( search );
+        }
+
+
+        @Override
+        public Observable<String> getEdgeTypesToTarget( final SearchEdgeType search ) {
+            return graphManager.getEdgeTypesToTarget( search );
+        }
+
+
+        @Override
+        public Observable<String> getIdTypesToTarget( final SearchIdType search ) {
+            return graphManager.getIdTypesToTarget( search );
+        }
+
+
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e9d652dd/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardingIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardingIT.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardingIT.java
index 01e07b1..4fe3098 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardingIT.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardingIT.java
@@ -34,7 +34,10 @@ import org.apache.usergrid.persistence.core.cassandra.ITRunner;
 import org.apache.usergrid.persistence.core.scope.ApplicationScope;
 import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
 import org.apache.usergrid.persistence.graph.guice.TestGraphModule;
+import org.apache.usergrid.persistence.graph.serialization.impl.shard.DirectedEdgeMeta;
 import org.apache.usergrid.persistence.graph.serialization.impl.shard.NodeShardApproximation;
+import org.apache.usergrid.persistence.graph.serialization.impl.shard.NodeType;
+import org.apache.usergrid.persistence.graph.serialization.impl.shard.Shard;
 import org.apache.usergrid.persistence.model.entity.Id;
 import org.apache.usergrid.persistence.model.util.UUIDGenerator;
 
@@ -101,7 +104,7 @@ public class GraphManagerShardingIT {
         //each edge causes 4 counts
         final long writeCount = flushCount/4;
 
-        assertTrue( "Shard size must be >= flush Count", maxShardSize >= flushCount );
+        assertTrue( "Shard size must be >= beginFlush Count", maxShardSize >= flushCount );
 
         Id targetId = null;
 
@@ -115,14 +118,21 @@ public class GraphManagerShardingIT {
         }
 
 
-        long shardCount = nodeShardApproximation.getCount( scope, sourceId, 0l, edgeType );
+
+        final DirectedEdgeMeta sourceEdgeMeta = DirectedEdgeMeta.fromSourceNodeTargetType(sourceId,  edgeType, targetId.getType() );
+        final Shard shard = new Shard(0, 0, true);
+
+
+        long shardCount = nodeShardApproximation.getCount( scope, shard, sourceEdgeMeta );
 
         assertEquals("Shard count for source node should be the same as write count", writeCount, shardCount);
 
 
         //now verify it's correct for the target
+        final DirectedEdgeMeta targetEdgeMeta = DirectedEdgeMeta.fromTargetNodeSourceType(targetId,  edgeType, sourceId.getType() );
+
 
-        shardCount = nodeShardApproximation.getCount( scope, targetId, 0l, edgeType );
+        shardCount = nodeShardApproximation.getCount( scope, shard, targetEdgeMeta );
 
         assertEquals(1, shardCount);
 
@@ -144,14 +154,10 @@ public class GraphManagerShardingIT {
         final long maxShardSize = graphFig.getShardSize();
 
 
-
-
-        final long startTime = System.currentTimeMillis();
-
-        //each edge causes 4 counts
+         //each edge causes 4 counts
         final long writeCount = flushCount/4;
 
-        assertTrue( "Shard size must be >= flush Count", maxShardSize >= flushCount );
+        assertTrue( "Shard size must be >= beginFlush Count", maxShardSize >= flushCount );
 
         Id sourceId = null;
 
@@ -165,14 +171,27 @@ public class GraphManagerShardingIT {
         }
 
 
-        long shardCount = nodeShardApproximation.getCount( scope, targetId, 0l, edgeType );
+        //this is from target->source, since the target id doesn't change
+        final DirectedEdgeMeta targetMeta = DirectedEdgeMeta.fromTargetNode( targetId, edgeType );
+        final Shard shard = new Shard(0l, 0l, true);
+
+        long targetWithType = nodeShardApproximation.getCount( scope, shard, targetMeta );
+
+        assertEquals("Shard count for target node should be the same as write count", writeCount, targetWithType);
 
-        assertEquals("Shard count for source node should be the same as write count", writeCount, shardCount);
+
+        final DirectedEdgeMeta targetNodeSource = DirectedEdgeMeta.fromTargetNodeSourceType( targetId, edgeType, "source" );
+
+        long shardCount = nodeShardApproximation.getCount( scope, shard, targetNodeSource );
+
+        assertEquals("Shard count for target node should be the same as write count", writeCount, shardCount);
 
 
         //now verify it's correct for the target
 
-        shardCount = nodeShardApproximation.getCount( scope, sourceId, 0l, edgeType );
+        final DirectedEdgeMeta sourceMeta = DirectedEdgeMeta.fromSourceNode( sourceId, edgeType );
+
+        shardCount = nodeShardApproximation.getCount( scope, shard, sourceMeta );
 
         assertEquals(1, shardCount);
 

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e9d652dd/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/EdgeShardSerializationTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/EdgeShardSerializationTest.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/EdgeShardSerializationTest.java
index 479e1bf..da57b0a 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/EdgeShardSerializationTest.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/EdgeShardSerializationTest.java
@@ -29,10 +29,10 @@ import org.junit.Rule;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 
+import org.apache.usergrid.persistence.collection.guice.MigrationManagerRule;
+import org.apache.usergrid.persistence.core.cassandra.CassandraRule;
 import org.apache.usergrid.persistence.core.cassandra.ITRunner;
 import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.core.cassandra.CassandraRule;
-import org.apache.usergrid.persistence.collection.guice.MigrationManagerRule;
 import org.apache.usergrid.persistence.graph.guice.TestGraphModule;
 import org.apache.usergrid.persistence.model.entity.Id;
 import org.apache.usergrid.persistence.model.util.UUIDGenerator;
@@ -49,8 +49,8 @@ import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
 
-@RunWith( ITRunner.class )
-@UseModules( { TestGraphModule.class } )
+@RunWith(ITRunner.class)
+@UseModules({ TestGraphModule.class })
 public class EdgeShardSerializationTest {
 
     @ClassRule
@@ -86,39 +86,54 @@ public class EdgeShardSerializationTest {
 
         final Id now = createId( "test" );
 
-        final long slice1 = 1000l;
+        final long timestamp = 10000l;
+
+        final Shard shard1 = new Shard( 1000l, timestamp, false );
 
-        final long slice2 = slice1 * 2;
+        final Shard shard2 = new Shard( shard1.getShardIndex() * 2, timestamp, true );
 
-        final long slice3 = slice2 * 2;
+        final Shard shard3 = new Shard( shard2.getShardIndex() * 2, timestamp, false );
 
-        String[] types = { "edgeType", "subType" };
+        final DirectedEdgeMeta sourceEdgeMeta = DirectedEdgeMeta.fromSourceNodeTargetType(now,  "edgeType", "subType"  );
 
-        MutationBatch batch = edgeShardSerialization.writeEdgeMeta( scope, now, slice1, types );
+        MutationBatch batch = edgeShardSerialization.writeShardMeta( scope, shard1, sourceEdgeMeta  );
 
-        batch.mergeShallow( edgeShardSerialization.writeEdgeMeta( scope, now, slice2, types ) );
+        batch.mergeShallow( edgeShardSerialization.writeShardMeta( scope, shard2, sourceEdgeMeta ) );
 
-        batch.mergeShallow( edgeShardSerialization.writeEdgeMeta( scope, now, slice3, types ) );
+        batch.mergeShallow( edgeShardSerialization.writeShardMeta( scope, shard3, sourceEdgeMeta ) );
 
         batch.execute();
 
 
-        Iterator<Long> results = edgeShardSerialization.getEdgeMetaData( scope, now, Optional.<Long>absent(), types );
+        Iterator<Shard> results =
+                edgeShardSerialization.getShardMetaData( scope, Optional.<Shard>absent(), sourceEdgeMeta );
+
 
-        assertEquals( slice3, results.next().longValue() );
+        assertEquals( shard3, results.next() );
 
-        assertEquals( slice2, results.next().longValue() );
+        assertEquals( shard2, results.next() );
 
-        assertEquals( slice1, results.next().longValue() );
+        assertEquals( shard1, results.next() );
+
+
+        assertFalse( results.hasNext() );
+
+        final DirectedEdgeMeta targetEdgeMeta = DirectedEdgeMeta.fromTargetNodeSourceType( now,  "edgeType", "subType"  );
+
+        //test we get nothing with the other node type
+        results =
+                edgeShardSerialization.getShardMetaData( scope, Optional.<Shard>absent(), targetEdgeMeta );
 
         assertFalse( results.hasNext() );
 
+
         //test paging and size
-        results = edgeShardSerialization.getEdgeMetaData( scope, now, Optional.of( slice2 ), types );
+        results = edgeShardSerialization.getShardMetaData( scope, Optional.of( shard2 ), sourceEdgeMeta );
 
-        assertEquals( slice2, results.next().longValue() );
+        assertEquals( shard2, results.next() );
 
-        assertEquals( slice1, results.next().longValue() );
+
+        assertEquals( shard1, results.next() );
 
 
         assertFalse( results.hasNext() );
@@ -130,50 +145,71 @@ public class EdgeShardSerializationTest {
 
         final Id now = createId( "test" );
 
-        final long slice1 = 1000l;
 
-        final long slice2 = slice1 * 2;
+        final long timestamp = 10000l;
+
+        final Shard shard1 = new Shard( 1000l, timestamp, false );
+
+        final Shard shard2 = new Shard( shard1.getShardIndex() * 2, timestamp, true );
 
-        final long slice3 = slice2 * 2;
+        final Shard shard3 = new Shard( shard2.getShardIndex() * 2, timestamp, false );
 
-        String[] types = { "edgeType", "subType" };
 
-        MutationBatch batch = edgeShardSerialization.writeEdgeMeta( scope, now, slice1, types );
+        final DirectedEdgeMeta sourceEdgeMeta = DirectedEdgeMeta.fromSourceNodeTargetType(now,  "edgeType", "subType"  );
 
-        batch.mergeShallow( edgeShardSerialization.writeEdgeMeta( scope, now, slice2, types ) );
 
-        batch.mergeShallow( edgeShardSerialization.writeEdgeMeta( scope, now, slice3, types ) );
+        MutationBatch batch =
+                edgeShardSerialization.writeShardMeta( scope, shard1, sourceEdgeMeta );
+
+        batch.mergeShallow(
+                edgeShardSerialization.writeShardMeta( scope, shard2, sourceEdgeMeta ) );
+
+        batch.mergeShallow(
+                edgeShardSerialization.writeShardMeta( scope, shard3, sourceEdgeMeta ) );
 
         batch.execute();
 
 
-        Iterator<Long> results = edgeShardSerialization.getEdgeMetaData( scope, now, Optional.<Long>absent(), types );
+        Iterator<Shard> results =
+                edgeShardSerialization.getShardMetaData( scope, Optional.<Shard>absent(), sourceEdgeMeta );
 
-        assertEquals( slice3, results.next().longValue() );
+        assertEquals( shard3, results.next() );
 
-        assertEquals( slice2, results.next().longValue() );
+        assertEquals( shard2, results.next() );
 
-        assertEquals( slice1, results.next().longValue() );
+        assertEquals( shard1, results.next() );
 
         assertFalse( results.hasNext() );
 
+        //test nothing with other type
+
+        final DirectedEdgeMeta targetEdgeMeta = DirectedEdgeMeta.fromTargetNodeSourceType( now,  "edgeType", "subType"  );
+
+        results =
+                edgeShardSerialization.getShardMetaData( scope, Optional.<Shard>absent(), targetEdgeMeta );
+
+        assertFalse( results.hasNext() );
+
+
         //test paging and size
-        edgeShardSerialization.removeEdgeMeta( scope, now, slice1, types ).execute();
+        edgeShardSerialization.removeShardMeta( scope, shard1, sourceEdgeMeta ).execute();
 
-        results = edgeShardSerialization.getEdgeMetaData( scope, now,Optional.<Long>absent(), types );
+        results =
+                edgeShardSerialization.getShardMetaData( scope, Optional.<Shard>absent(), sourceEdgeMeta );
 
-        assertEquals( slice3, results.next().longValue() );
+        assertEquals( shard3, results.next() );
 
-        assertEquals( slice2, results.next().longValue() );
+        assertEquals( shard2, results.next() );
 
         assertFalse( results.hasNext() );
 
 
-        edgeShardSerialization.removeEdgeMeta( scope, now, slice2, types ).execute();
+        edgeShardSerialization.removeShardMeta( scope, shard2, sourceEdgeMeta ).execute();
 
-        edgeShardSerialization.removeEdgeMeta( scope, now, slice3, types ).execute();
+        edgeShardSerialization.removeShardMeta( scope, shard3, sourceEdgeMeta ).execute();
 
-        results = edgeShardSerialization.getEdgeMetaData( scope, now, Optional.<Long>absent(), types );
+        results =
+                edgeShardSerialization.getShardMetaData( scope, Optional.<Shard>absent(), sourceEdgeMeta );
 
 
         assertFalse( results.hasNext() );