You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by to...@apache.org on 2014/08/14 17:20:22 UTC
[3/6] Implemented New rolling shard algorithm. This should allow
eventual shard consistency between all clients without the need for an
external locking allocation system
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e9d652dd/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardedEdgeSerializationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardedEdgeSerializationImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardedEdgeSerializationImpl.java
new file mode 100644
index 0000000..c33b835
--- /dev/null
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardedEdgeSerializationImpl.java
@@ -0,0 +1,656 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one
+ * * or more contributor license agreements. See the NOTICE file
+ * * distributed with this work for additional information
+ * * regarding copyright ownership. The ASF licenses this file
+ * * to you under the Apache License, Version 2.0 (the
+ * * "License"); you may not use this file except in compliance
+ * * with the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing,
+ * * software distributed under the License is distributed on an
+ * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * * KIND, either express or implied. See the License for the
+ * * specific language governing permissions and limitations
+ * * under the License.
+ *
+ */
+package org.apache.usergrid.persistence.graph.serialization.impl.shard.impl;
+
+
+import java.util.Iterator;
+import java.util.UUID;
+
+import javax.inject.Inject;
+
+import org.apache.usergrid.persistence.core.astyanax.CassandraConfig;
+import org.apache.usergrid.persistence.core.astyanax.MultiTennantColumnFamily;
+import org.apache.usergrid.persistence.core.astyanax.ScopedRowKey;
+import org.apache.usergrid.persistence.core.consistency.TimeService;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.core.util.ValidationUtils;
+import org.apache.usergrid.persistence.graph.Edge;
+import org.apache.usergrid.persistence.graph.GraphFig;
+import org.apache.usergrid.persistence.graph.MarkedEdge;
+import org.apache.usergrid.persistence.graph.SearchByEdge;
+import org.apache.usergrid.persistence.graph.SearchByEdgeType;
+import org.apache.usergrid.persistence.graph.SearchByIdType;
+import org.apache.usergrid.persistence.graph.impl.SimpleMarkedEdge;
+import org.apache.usergrid.persistence.graph.serialization.impl.shard.DirectedEdge;
+import org.apache.usergrid.persistence.graph.serialization.impl.shard.DirectedEdgeMeta;
+import org.apache.usergrid.persistence.graph.serialization.impl.shard.EdgeColumnFamilies;
+import org.apache.usergrid.persistence.graph.serialization.impl.shard.EdgeRowKey;
+import org.apache.usergrid.persistence.graph.serialization.impl.shard.EdgeShardStrategy;
+import org.apache.usergrid.persistence.graph.serialization.impl.shard.NodeType;
+import org.apache.usergrid.persistence.graph.serialization.impl.shard.RowKey;
+import org.apache.usergrid.persistence.graph.serialization.impl.shard.RowKeyType;
+import org.apache.usergrid.persistence.graph.serialization.impl.shard.Shard;
+import org.apache.usergrid.persistence.graph.serialization.impl.shard.ShardEntryGroup;
+import org.apache.usergrid.persistence.graph.serialization.impl.shard.ShardedEdgeSerialization;
+import org.apache.usergrid.persistence.graph.serialization.util.EdgeUtils;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.fasterxml.uuid.UUIDComparator;
+import com.fasterxml.uuid.impl.UUIDUtil;
+import com.google.common.base.Optional;
+import com.google.inject.Singleton;
+import com.netflix.astyanax.Keyspace;
+import com.netflix.astyanax.MutationBatch;
+import com.netflix.astyanax.Serializer;
+import com.netflix.astyanax.util.RangeBuilder;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+
+@Singleton
+public class ShardedEdgeSerializationImpl implements ShardedEdgeSerialization {
+
+ protected final Keyspace keyspace;
+ protected final CassandraConfig cassandraConfig;
+ protected final GraphFig graphFig;
+ protected final EdgeShardStrategy writeEdgeShardStrategy;
+ protected final TimeService timeService;
+
+
+ @Inject
+ public ShardedEdgeSerializationImpl( final Keyspace keyspace, final CassandraConfig cassandraConfig,
+ final GraphFig graphFig, final EdgeShardStrategy writeEdgeShardStrategy,
+ final TimeService timeService ) {
+
+
+ checkNotNull( "keyspace required", keyspace );
+ checkNotNull( "cassandraConfig required", cassandraConfig );
+ checkNotNull( "consistencyFig required", graphFig );
+ checkNotNull( "writeEdgeShardStrategy required", writeEdgeShardStrategy );
+ checkNotNull( "timeService required", timeService );
+
+
+ this.keyspace = keyspace;
+ this.cassandraConfig = cassandraConfig;
+ this.graphFig = graphFig;
+ this.writeEdgeShardStrategy = writeEdgeShardStrategy;
+ this.timeService = timeService;
+ }
+
+
+ @Override
+ public MutationBatch writeEdge( final EdgeColumnFamilies columnFamilies, final ApplicationScope scope,
+ final MarkedEdge markedEdge, final UUID timestamp ) {
+ ValidationUtils.validateApplicationScope( scope );
+ EdgeUtils.validateEdge( markedEdge );
+ ValidationUtils.verifyTimeUuid( timestamp, "timestamp" );
+
+
+ final MutationBatch batch = keyspace.prepareMutationBatch().withConsistencyLevel( cassandraConfig.getWriteCL() )
+ .withTimestamp( timestamp.timestamp() );
+
+ final boolean isDeleted = markedEdge.isDeleted();
+
+
+ doWrite( columnFamilies, scope, markedEdge, new RowOp<RowKey>() {
+ @Override
+ public void writeEdge( final MultiTennantColumnFamily<ApplicationScope, RowKey, DirectedEdge> columnFamily,
+ final RowKey rowKey, final DirectedEdge edge ) {
+ batch.withRow( columnFamily, ScopedRowKey.fromKey( scope, rowKey ) ).putColumn( edge, isDeleted );
+ }
+
+
+ @Override
+ public void countEdge( final Shard shard, final DirectedEdgeMeta directedEdgeMeta ) {
+ if(!isDeleted) {
+ writeEdgeShardStrategy.increment( scope, shard, 1, directedEdgeMeta );
+ }
+ }
+
+
+
+ @Override
+ public void writeVersion( final MultiTennantColumnFamily<ApplicationScope, EdgeRowKey, Long> columnFamily,
+ final EdgeRowKey rowKey, final long timestamp ) {
+ batch.withRow( columnFamily, ScopedRowKey.fromKey( scope, rowKey ) ).putColumn( timestamp, isDeleted );
+ }
+ } );
+
+
+ return batch;
+ }
+
+
+ @Override
+ public MutationBatch deleteEdge( final EdgeColumnFamilies columnFamilies, final ApplicationScope scope,
+ final MarkedEdge markedEdge, final UUID timestamp ) {
+ ValidationUtils.validateApplicationScope( scope );
+ EdgeUtils.validateEdge( markedEdge );
+ ValidationUtils.verifyTimeUuid( timestamp, "timestamp" );
+
+
+ final MutationBatch batch = keyspace.prepareMutationBatch().withConsistencyLevel( cassandraConfig.getWriteCL() )
+ .withTimestamp( timestamp.timestamp() );
+
+
+ doWrite( columnFamilies, scope, markedEdge, new RowOp<RowKey>() {
+ @Override
+ public void writeEdge( final MultiTennantColumnFamily<ApplicationScope, RowKey, DirectedEdge> columnFamily,
+ final RowKey rowKey, final DirectedEdge edge ) {
+ batch.withRow( columnFamily, ScopedRowKey.fromKey( scope, rowKey ) ).deleteColumn( edge );
+ }
+
+
+ @Override
+ public void countEdge( final Shard shard, final DirectedEdgeMeta directedEdgeMeta ) {
+ writeEdgeShardStrategy.increment( scope, shard, -1, directedEdgeMeta );
+ }
+
+
+ @Override
+ public void writeVersion( final MultiTennantColumnFamily<ApplicationScope, EdgeRowKey, Long> columnFamily,
+ final EdgeRowKey rowKey, final long timestamp ) {
+ batch.withRow( columnFamily, ScopedRowKey.fromKey( scope, rowKey ) ).deleteColumn( timestamp );
+ }
+ } );
+
+
+ return batch;
+ }
+
+
+ /**
+ * EdgeWrite the edges internally
+ *
+ * @param scope The scope to encapsulate
+ * @param edge The edge to write
+ * @param op The row operation to invoke
+ */
+ private void doWrite( final EdgeColumnFamilies columnFamilies, final ApplicationScope scope, final MarkedEdge edge,
+ final RowOp op ) {
+ ValidationUtils.validateApplicationScope( scope );
+ EdgeUtils.validateEdge( edge );
+
+ final Id sourceNodeId = edge.getSourceNode();
+ final String sourceNodeType = sourceNodeId.getType();
+ final Id targetNodeId = edge.getTargetNode();
+ final String targetNodeType = targetNodeId.getType();
+ final long timestamp = edge.getTimestamp();
+ final String type = edge.getType();
+
+
+ /**
+ * Key in the serializers based on the edge
+ */
+
+
+ /**
+ * write edges from source->target
+ */
+
+
+ final long time = timeService.getCurrentTime();
+
+ final DirectedEdge sourceEdge = new DirectedEdge( targetNodeId, timestamp );
+
+ final DirectedEdgeMeta sourceEdgeMeta = DirectedEdgeMeta.fromSourceNode( sourceNodeId, type );
+
+
+
+ final ShardEntryGroup sourceRowKeyShard =
+ writeEdgeShardStrategy.getWriteShards( scope, timestamp, sourceEdgeMeta );
+
+ final MultiTennantColumnFamily<ApplicationScope, RowKey, DirectedEdge> sourceCf =
+ columnFamilies.getSourceNodeCfName();
+
+
+ for ( Shard shard : sourceRowKeyShard.getWriteShards(time) ) {
+
+ final long shardId = shard.getShardIndex();
+ final RowKey sourceRowKey = new RowKey( sourceNodeId, type, shardId );
+ op.writeEdge( sourceCf, sourceRowKey, sourceEdge );
+ op.countEdge( shard, sourceEdgeMeta );
+ }
+
+
+
+ final DirectedEdgeMeta sourceEdgeTargetTypeMeta = DirectedEdgeMeta.fromSourceNodeTargetType( sourceNodeId, type, targetNodeType );
+
+
+ final ShardEntryGroup sourceWithTypeRowKeyShard = writeEdgeShardStrategy
+ .getWriteShards( scope, timestamp, sourceEdgeTargetTypeMeta );
+
+ final MultiTennantColumnFamily<ApplicationScope, RowKeyType, DirectedEdge> targetCf =
+ columnFamilies.getSourceNodeTargetTypeCfName();
+
+ for ( Shard shard : sourceWithTypeRowKeyShard.getWriteShards(time) ) {
+
+ final long shardId = shard.getShardIndex();
+ final RowKeyType sourceRowKeyType = new RowKeyType( sourceNodeId, type, targetNodeId, shardId );
+
+ op.writeEdge( targetCf, sourceRowKeyType, sourceEdge );
+ op.countEdge( shard, sourceEdgeTargetTypeMeta );
+ }
+
+
+ /**
+ * write edges from target<-source
+ */
+
+ final DirectedEdge targetEdge = new DirectedEdge( sourceNodeId, timestamp );
+
+ final DirectedEdgeMeta targetEdgeMeta = DirectedEdgeMeta.fromTargetNode( targetNodeId, type );
+
+
+
+ final ShardEntryGroup targetRowKeyShard =
+ writeEdgeShardStrategy.getWriteShards( scope, timestamp, targetEdgeMeta );
+
+ final MultiTennantColumnFamily<ApplicationScope, RowKey, DirectedEdge> sourceByTargetCf =
+ columnFamilies.getTargetNodeCfName();
+
+ for ( Shard shard : targetRowKeyShard.getWriteShards(time) ) {
+ final long shardId = shard.getShardIndex();
+ final RowKey targetRowKey = new RowKey( targetNodeId, type, shardId );
+
+ op.writeEdge( sourceByTargetCf, targetRowKey, targetEdge );
+ op.countEdge( shard, targetEdgeMeta );
+ }
+
+
+
+ final DirectedEdgeMeta targetEdgeSourceTypeMeta = DirectedEdgeMeta.fromTargetNodeSourceType( targetNodeId, type, sourceNodeType );
+
+
+ final ShardEntryGroup targetWithTypeRowKeyShard = writeEdgeShardStrategy
+ .getWriteShards( scope, timestamp, targetEdgeSourceTypeMeta );
+
+ final MultiTennantColumnFamily<ApplicationScope, RowKeyType, DirectedEdge> targetBySourceCf =
+ columnFamilies.getTargetNodeSourceTypeCfName();
+
+
+ for ( Shard shard : targetWithTypeRowKeyShard.getWriteShards(time) ) {
+
+ final long shardId = shard.getShardIndex();
+
+ final RowKeyType targetRowKeyType = new RowKeyType( targetNodeId, type, sourceNodeId, shardId );
+
+
+ op.writeEdge( targetBySourceCf, targetRowKeyType, targetEdge );
+ op.countEdge( shard, targetEdgeSourceTypeMeta );
+ }
+
+ /**
+ * Always a 0l shard, we're hard limiting 2b timestamps for the same edge
+ */
+ final EdgeRowKey edgeRowKey = new EdgeRowKey( sourceNodeId, type, targetNodeId, 0l );
+
+
+ /**
+ * Write this in the timestamp log for this edge of source->target
+ */
+ op.writeVersion( columnFamilies.getGraphEdgeVersions(), edgeRowKey, timestamp );
+ }
+
+
+ @Override
+ public Iterator<MarkedEdge> getEdgeVersions( final EdgeColumnFamilies columnFamilies, final ApplicationScope scope,
+ final SearchByEdge search, final Iterator<ShardEntryGroup> shards ) {
+ ValidationUtils.validateApplicationScope( scope );
+ EdgeUtils.validateSearchByEdge( search );
+
+ final Id targetId = search.targetNode();
+ final Id sourceId = search.sourceNode();
+ final String type = search.getType();
+ final long maxTimestamp = search.getMaxTimestamp();
+ final MultiTennantColumnFamily<ApplicationScope, EdgeRowKey, Long> columnFamily =
+ columnFamilies.getGraphEdgeVersions();
+ final Serializer<Long> serializer = columnFamily.getColumnSerializer();
+
+ final EdgeSearcher<EdgeRowKey, Long, MarkedEdge> searcher =
+ new EdgeSearcher<EdgeRowKey, Long, MarkedEdge>( scope, maxTimestamp, search.last(), shards ) {
+
+ @Override
+ protected Serializer<Long> getSerializer() {
+ return serializer;
+ }
+
+
+ @Override
+ public void setRange( final RangeBuilder builder ) {
+
+
+ if ( last.isPresent() ) {
+ super.setRange( builder );
+ return;
+ }
+
+ //start seeking at a value < our max version
+ builder.setStart( maxTimestamp );
+ }
+
+
+ @Override
+ protected EdgeRowKey generateRowKey( long shard ) {
+ return new EdgeRowKey( sourceId, type, targetId, shard );
+ }
+
+
+ @Override
+ protected Long getStartColumn( final Edge last ) {
+ return last.getTimestamp();
+ }
+
+
+ @Override
+ protected MarkedEdge createEdge( final Long column, final boolean marked ) {
+ return new SimpleMarkedEdge( sourceId, type, targetId, column.longValue(), marked );
+ }
+
+
+ @Override
+ public int compare( final MarkedEdge o1, final MarkedEdge o2 ) {
+ return Long.compare( o1.getTimestamp(), o2.getTimestamp() );
+ }
+ };
+
+ return new ShardRowIterator<>( searcher, columnFamily, keyspace, cassandraConfig.getReadCL(),
+ graphFig.getScanPageSize() );
+ }
+
+
+ @Override
+ public Iterator<MarkedEdge> getEdgesFromSource( final EdgeColumnFamilies columnFamilies,
+ final ApplicationScope scope, final SearchByEdgeType edgeType,
+ final Iterator<ShardEntryGroup> shards ) {
+
+ ValidationUtils.validateApplicationScope( scope );
+ EdgeUtils.validateSearchByEdgeType( edgeType );
+
+ final Id sourceId = edgeType.getNode();
+ final String type = edgeType.getType();
+ final long maxTimestamp = edgeType.getMaxTimestamp();
+ final MultiTennantColumnFamily<ApplicationScope, RowKey, DirectedEdge> columnFamily =
+ columnFamilies.getSourceNodeCfName();
+ final Serializer<DirectedEdge> serializer = columnFamily.getColumnSerializer();
+
+
+ final SourceEdgeSearcher<RowKey, DirectedEdge, MarkedEdge> searcher =
+ new SourceEdgeSearcher<RowKey, DirectedEdge, MarkedEdge>( scope, maxTimestamp, edgeType.last(), shards ) {
+
+
+ @Override
+ protected Serializer<DirectedEdge> getSerializer() {
+ return serializer;
+ }
+
+
+ @Override
+ protected RowKey generateRowKey( long shard ) {
+ return new RowKey( sourceId, type, shard );
+ }
+
+
+ @Override
+ protected DirectedEdge getStartColumn( final Edge last ) {
+ return new DirectedEdge( last.getTargetNode(), last.getTimestamp() );
+ }
+
+
+ @Override
+ protected MarkedEdge createEdge( final DirectedEdge edge, final boolean marked ) {
+ return new SimpleMarkedEdge( sourceId, type, edge.id, edge.timestamp, marked );
+ }
+
+
+ };
+
+
+ return new ShardRowIterator<>( searcher, columnFamily, keyspace, cassandraConfig.getReadCL(),
+ graphFig.getScanPageSize() );
+ }
+
+
+ @Override
+ public Iterator<MarkedEdge> getEdgesFromSourceByTargetType( final EdgeColumnFamilies columnFamilies,
+ final ApplicationScope scope,
+ final SearchByIdType edgeType,
+ final Iterator<ShardEntryGroup> shards ) {
+
+ ValidationUtils.validateApplicationScope( scope );
+ EdgeUtils.validateSearchByEdgeType( edgeType );
+
+ final Id targetId = edgeType.getNode();
+ final String type = edgeType.getType();
+ final String targetType = edgeType.getIdType();
+ final long maxTimestamp = edgeType.getMaxTimestamp();
+ final MultiTennantColumnFamily<ApplicationScope, RowKeyType, DirectedEdge> columnFamily =
+ columnFamilies.getSourceNodeTargetTypeCfName();
+ final Serializer<DirectedEdge> serializer = columnFamily.getColumnSerializer();
+
+
+ final SourceEdgeSearcher<RowKeyType, DirectedEdge, MarkedEdge> searcher =
+ new SourceEdgeSearcher<RowKeyType, DirectedEdge, MarkedEdge>( scope, maxTimestamp, edgeType.last(), shards ) {
+
+ @Override
+ protected Serializer<DirectedEdge> getSerializer() {
+ return serializer;
+ }
+
+
+ @Override
+ protected RowKeyType generateRowKey( long shard ) {
+ return new RowKeyType( targetId, type, targetType, shard );
+ }
+
+
+ @Override
+ protected DirectedEdge getStartColumn( final Edge last ) {
+ return new DirectedEdge( last.getTargetNode(), last.getTimestamp() );
+ }
+
+
+ @Override
+ protected MarkedEdge createEdge( final DirectedEdge edge, final boolean marked ) {
+ return new SimpleMarkedEdge( targetId, type, edge.id, edge.timestamp, marked );
+ }
+
+
+
+ };
+
+ return new ShardRowIterator( searcher, columnFamily, keyspace, cassandraConfig.getReadCL(),
+ graphFig.getScanPageSize() );
+ }
+
+
+
+
+ @Override
+ public Iterator<MarkedEdge> getEdgesToTarget( final EdgeColumnFamilies columnFamilies, final ApplicationScope scope,
+ final SearchByEdgeType edgeType,
+ final Iterator<ShardEntryGroup> shards ) {
+ ValidationUtils.validateApplicationScope( scope );
+ EdgeUtils.validateSearchByEdgeType( edgeType );
+
+ final Id targetId = edgeType.getNode();
+ final String type = edgeType.getType();
+ final long maxTimestamp = edgeType.getMaxTimestamp();
+ final MultiTennantColumnFamily<ApplicationScope, RowKey, DirectedEdge> columnFamily =
+ columnFamilies.getTargetNodeCfName();
+ final Serializer<DirectedEdge> serializer = columnFamily.getColumnSerializer();
+
+ final TargetEdgeSearcher<RowKey, DirectedEdge, MarkedEdge> searcher =
+ new TargetEdgeSearcher<RowKey, DirectedEdge, MarkedEdge>( scope, maxTimestamp, edgeType.last(), shards ) {
+
+ @Override
+ protected Serializer<DirectedEdge> getSerializer() {
+ return serializer;
+ }
+
+
+ @Override
+ protected RowKey generateRowKey( long shard ) {
+ return new RowKey( targetId, type, shard );
+ }
+
+
+ @Override
+ protected DirectedEdge getStartColumn( final Edge last ) {
+ return new DirectedEdge( last.getSourceNode(), last.getTimestamp() );
+ }
+
+
+ @Override
+ protected MarkedEdge createEdge( final DirectedEdge edge, final boolean marked ) {
+ return new SimpleMarkedEdge( edge.id, type, targetId, edge.timestamp, marked );
+ }
+ };
+
+
+ return new ShardRowIterator<>( searcher, columnFamily, keyspace, cassandraConfig.getReadCL(),
+ graphFig.getScanPageSize() );
+ }
+
+
+ @Override
+ public Iterator<MarkedEdge> getEdgesToTargetBySourceType( final EdgeColumnFamilies columnFamilies,
+ final ApplicationScope scope,
+ final SearchByIdType edgeType,
+ final Iterator<ShardEntryGroup> shards ) {
+
+ ValidationUtils.validateApplicationScope( scope );
+ EdgeUtils.validateSearchByEdgeType( edgeType );
+
+ final Id targetId = edgeType.getNode();
+ final String sourceType = edgeType.getIdType();
+ final String type = edgeType.getType();
+ final long maxTimestamp = edgeType.getMaxTimestamp();
+ final MultiTennantColumnFamily<ApplicationScope, RowKeyType, DirectedEdge> columnFamily =
+ columnFamilies.getTargetNodeSourceTypeCfName();
+ final Serializer<DirectedEdge> serializer = columnFamily.getColumnSerializer();
+
+
+ final TargetEdgeSearcher<RowKeyType, DirectedEdge, MarkedEdge> searcher =
+ new TargetEdgeSearcher<RowKeyType, DirectedEdge, MarkedEdge>( scope, maxTimestamp, edgeType.last(), shards ) {
+ @Override
+ protected Serializer<DirectedEdge> getSerializer() {
+ return serializer;
+ }
+
+
+ @Override
+ protected RowKeyType generateRowKey( final long shard ) {
+ return new RowKeyType( targetId, type, sourceType, shard );
+ }
+
+
+ @Override
+ protected DirectedEdge getStartColumn( final Edge last ) {
+ return new DirectedEdge( last.getTargetNode(), last.getTimestamp() );
+ }
+
+
+ @Override
+ protected MarkedEdge createEdge( final DirectedEdge edge, final boolean marked ) {
+ return new SimpleMarkedEdge( edge.id, type, targetId, edge.timestamp, marked );
+ }
+ };
+
+ return new ShardRowIterator<>( searcher, columnFamily, keyspace, cassandraConfig.getReadCL(),
+ graphFig.getScanPageSize() );
+ }
+
+
+ /**
+ * Class for performing searched on rows based on source id
+ */
+ private static abstract class SourceEdgeSearcher<R, C, T extends Edge> extends EdgeSearcher<R, C, T>{
+
+ protected SourceEdgeSearcher( final ApplicationScope scope, final long maxTimestamp, final Optional<Edge> last,
+ final Iterator<ShardEntryGroup> shards ) {
+ super( scope, maxTimestamp, last, shards );
+ }
+
+
+ public int compare( final T o1, final T o2 ) {
+ int compare = Long.compare(o1.getTimestamp(), o2.getTimestamp());
+
+ if(compare == 0){
+ compare = o1.getTargetNode().compareTo( o2.getTargetNode());
+ }
+
+ return compare;
+ }
+
+
+ }
+
+
+ /**
+ * Class for performing searched on rows based on target id
+ */
+ private static abstract class TargetEdgeSearcher<R, C, T extends Edge> extends EdgeSearcher<R, C, T>{
+
+ protected TargetEdgeSearcher( final ApplicationScope scope, final long maxTimestamp, final Optional<Edge> last,
+ final Iterator<ShardEntryGroup> shards ) {
+ super( scope, maxTimestamp, last, shards );
+ }
+
+
+ public int compare( final T o1, final T o2 ) {
+ int compare = Long.compare(o1.getTimestamp(), o2.getTimestamp());
+
+ if(compare == 0){
+ compare = o1.getTargetNode().compareTo( o2.getTargetNode());
+ }
+
+ return compare;
+ }
+
+
+ }
+
+ /**
+ * Simple callback to perform puts and deletes with a common row setup code
+ *
+ * @param <R> The row key type
+ */
+ private static interface RowOp<R> {
+
+ /**
+ * Write the edge with the given data
+ */
+ void writeEdge( final MultiTennantColumnFamily<ApplicationScope, R, DirectedEdge> columnFamily, final R rowKey,
+ final DirectedEdge edge );
+
+ /**
+ * Perform the count on the edge
+ */
+ void countEdge( final Shard shard, final DirectedEdgeMeta directedEdgeMeta);
+
+ /**
+ * Write the edge into the version cf
+ */
+ void writeVersion( final MultiTennantColumnFamily<ApplicationScope, EdgeRowKey, Long> columnFamily,
+ final EdgeRowKey rowKey, long timestamp );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e9d652dd/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/SizebasedEdgeColumnFamilies.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/SizebasedEdgeColumnFamilies.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/SizebasedEdgeColumnFamilies.java
new file mode 100644
index 0000000..9050b0a
--- /dev/null
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/SizebasedEdgeColumnFamilies.java
@@ -0,0 +1,150 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one
+ * * or more contributor license agreements. See the NOTICE file
+ * * distributed with this work for additional information
+ * * regarding copyright ownership. The ASF licenses this file
+ * * to you under the Apache License, Version 2.0 (the
+ * * "License"); you may not use this file except in compliance
+ * * with the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing,
+ * * software distributed under the License is distributed on an
+ * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * * KIND, either express or implied. See the License for the
+ * * specific language governing permissions and limitations
+ * * under the License.
+ *
+ */
+package org.apache.usergrid.persistence.graph.serialization.impl.shard.impl;
+
+
+import java.util.Arrays;
+import java.util.Collection;
+
+import org.apache.cassandra.db.marshal.BytesType;
+import org.apache.cassandra.db.marshal.DynamicCompositeType;
+
+import org.apache.usergrid.persistence.core.astyanax.ColumnTypes;
+import org.apache.usergrid.persistence.core.astyanax.MultiTennantColumnFamily;
+import org.apache.usergrid.persistence.core.astyanax.MultiTennantColumnFamilyDefinition;
+import org.apache.usergrid.persistence.core.astyanax.OrganizationScopedRowKeySerializer;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.graph.serialization.impl.shard.DirectedEdge;
+import org.apache.usergrid.persistence.graph.serialization.impl.shard.EdgeColumnFamilies;
+import org.apache.usergrid.persistence.graph.serialization.impl.shard.EdgeRowKey;
+import org.apache.usergrid.persistence.graph.serialization.impl.shard.RowKey;
+import org.apache.usergrid.persistence.graph.serialization.impl.shard.RowKeyType;
+
+import com.netflix.astyanax.serializers.LongSerializer;
+
+import static org.apache.usergrid.persistence.core.astyanax.ColumnTypes.LONG_TYPE_REVERSED;
+import static org.apache.usergrid.persistence.core.astyanax.ColumnTypes.UUID_TYPE_REVERSED;
+
+
+/**
+ * Implementation of size based column family
+ */
+public class SizebasedEdgeColumnFamilies implements EdgeColumnFamilies {
+
+
+ //Row key with no type
+ private static final RowSerializer ROW_SERIALIZER = new RowSerializer();
+
+ //row key with target id type
+ private static final RowTypeSerializer ROW_TYPE_SERIALIZER = new RowTypeSerializer();
+
+ private static final EdgeRowKeySerializer EDGE_ROW_KEY_SERIALIZER = new EdgeRowKeySerializer();
+
+ //Edge serializers
+ private static final EdgeSerializer EDGE_SERIALIZER = new EdgeSerializer();
+
+ private static final LongSerializer LONG_SERIALIZER = LongSerializer.get();
+
+ private static final String EDGE_DYNAMIC_COMPOSITE_TYPE =
+ //we purposefully associate lower case "l" and "u" with reversed types. This way we can use
+ //the default serialization in Astayanax, but get reverse order in cassandra
+ DynamicCompositeType.class.getSimpleName() + "(s=>UTF8Type,l=>" + LONG_TYPE_REVERSED + ",u=>"
+ + UUID_TYPE_REVERSED + ")";
+
+
+ //initialize the CF's from our implementation
+ private static final MultiTennantColumnFamily<ApplicationScope, RowKey, DirectedEdge> SOURCE_NODE_EDGES =
+ new MultiTennantColumnFamily<>( "Graph_Source_Node_Edges",
+ new OrganizationScopedRowKeySerializer<>( ROW_SERIALIZER ), EDGE_SERIALIZER );
+
+
+ private static final MultiTennantColumnFamily<ApplicationScope, RowKey, DirectedEdge> TARGET_NODE_EDGES =
+ new MultiTennantColumnFamily<>( "Graph_Target_Node_Edges",
+ new OrganizationScopedRowKeySerializer<>( ROW_SERIALIZER ), EDGE_SERIALIZER );
+
+
+ private static final MultiTennantColumnFamily<ApplicationScope, RowKeyType, DirectedEdge> SOURCE_NODE_TARGET_TYPE =
+ new MultiTennantColumnFamily<>( "Graph_Source_Node_Target_Type",
+ new OrganizationScopedRowKeySerializer<>( ROW_TYPE_SERIALIZER ), EDGE_SERIALIZER );
+
+
+ /**
+ * The edges that are to the target node with the source type. The target node is the row key
+ */
+ private static final MultiTennantColumnFamily<ApplicationScope, RowKeyType, DirectedEdge> TARGET_NODE_SOURCE_TYPE =
+ new MultiTennantColumnFamily<>( "Graph_Target_Node_Source_Type",
+ new OrganizationScopedRowKeySerializer<>( ROW_TYPE_SERIALIZER ), EDGE_SERIALIZER );
+
+
+ private static final MultiTennantColumnFamily<ApplicationScope, EdgeRowKey, Long> EDGE_VERSIONS =
+ new MultiTennantColumnFamily<>( "Graph_Edge_Versions",
+ new OrganizationScopedRowKeySerializer<>( EDGE_ROW_KEY_SERIALIZER ), LONG_SERIALIZER );
+
+
+ @Override
+ public MultiTennantColumnFamily<ApplicationScope, RowKey, DirectedEdge> getSourceNodeCfName() {
+ return SOURCE_NODE_EDGES;
+ }
+
+
+ @Override
+ public MultiTennantColumnFamily<ApplicationScope, RowKey, DirectedEdge> getTargetNodeCfName() {
+ return TARGET_NODE_EDGES;
+ }
+
+
+ @Override
+ public MultiTennantColumnFamily<ApplicationScope, RowKeyType, DirectedEdge> getSourceNodeTargetTypeCfName() {
+ return SOURCE_NODE_TARGET_TYPE;
+ }
+
+
+ @Override
+ public MultiTennantColumnFamily<ApplicationScope, RowKeyType, DirectedEdge> getTargetNodeSourceTypeCfName() {
+ return TARGET_NODE_SOURCE_TYPE;
+ }
+
+
+ @Override
+ public MultiTennantColumnFamily<ApplicationScope, EdgeRowKey, Long> getGraphEdgeVersions() {
+ return EDGE_VERSIONS;
+ }
+
+
+ @Override
+ public Collection<MultiTennantColumnFamilyDefinition> getColumnFamilies() {
+ return Arrays
+ .asList( graphCf( SOURCE_NODE_EDGES ), graphCf( TARGET_NODE_EDGES ), graphCf( SOURCE_NODE_TARGET_TYPE ),
+ graphCf( TARGET_NODE_SOURCE_TYPE ),
+ new MultiTennantColumnFamilyDefinition( EDGE_VERSIONS, BytesType.class.getSimpleName(),
+ ColumnTypes.LONG_TYPE_REVERSED, BytesType.class.getSimpleName(),
+ MultiTennantColumnFamilyDefinition.CacheOption.KEYS ) );
+ }
+
+
+ /**
+ * Helper to generate an edge definition by the type
+ */
+ private MultiTennantColumnFamilyDefinition graphCf( MultiTennantColumnFamily cf ) {
+ return new MultiTennantColumnFamilyDefinition( cf, BytesType.class.getSimpleName(), EDGE_DYNAMIC_COMPOSITE_TYPE,
+ BytesType.class.getSimpleName(), MultiTennantColumnFamilyDefinition.CacheOption.KEYS );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e9d652dd/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/SizebasedEdgeShardStrategy.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/SizebasedEdgeShardStrategy.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/SizebasedEdgeShardStrategy.java
index f246f23..6558273 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/SizebasedEdgeShardStrategy.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/SizebasedEdgeShardStrategy.java
@@ -23,9 +23,14 @@ package org.apache.usergrid.persistence.graph.serialization.impl.shard.impl;
import java.util.Iterator;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.graph.serialization.impl.shard.DirectedEdgeMeta;
import org.apache.usergrid.persistence.graph.serialization.impl.shard.EdgeShardStrategy;
import org.apache.usergrid.persistence.graph.serialization.impl.shard.NodeShardApproximation;
import org.apache.usergrid.persistence.graph.serialization.impl.shard.NodeShardCache;
+import org.apache.usergrid.persistence.graph.serialization.impl.shard.NodeType;
+import org.apache.usergrid.persistence.graph.serialization.impl.shard.Shard;
+import org.apache.usergrid.persistence.graph.serialization.impl.shard.ShardEntryGroup;
+import org.apache.usergrid.persistence.graph.serialization.impl.shard.ShardOperator;
import org.apache.usergrid.persistence.model.entity.Id;
import com.google.inject.Inject;
@@ -52,52 +57,21 @@ public class SizebasedEdgeShardStrategy implements EdgeShardStrategy {
@Override
- public long getWriteShard( final ApplicationScope scope, final Id rowKeyId, final long timestamp,
- final String... types ) {
- return shardCache.getSlice( scope, rowKeyId, timestamp, types );
+ public ShardEntryGroup getWriteShards( final ApplicationScope scope,
+ final long timestamp, final DirectedEdgeMeta directedEdgeMeta ) {
+ return shardCache.getWriteShardGroup( scope, timestamp, directedEdgeMeta);
}
@Override
- public Iterator<Long> getReadShards( final ApplicationScope scope, final Id rowKeyId, final long maxTimestamp,
- final String... types ) {
- return shardCache.getVersions( scope, rowKeyId, maxTimestamp, types );
+ public Iterator<ShardEntryGroup> getReadShards( final ApplicationScope scope, final long maxTimestamp, final DirectedEdgeMeta directedEdgeMeta ) {
+ return shardCache.getReadShardGroup( scope, maxTimestamp, directedEdgeMeta );
}
@Override
- public void increment( final ApplicationScope scope, final Id rowKeyId, final long shardId, final long count,
- final String... types ) {
- shardApproximation.increment( scope, rowKeyId, shardId, count, types );
- }
-
-
- @Override
- public String getSourceNodeCfName() {
- return "Graph_Source_Node_Edges";
- }
-
-
- @Override
- public String getTargetNodeCfName() {
- return "Graph_Target_Node_Edges";
- }
-
-
- @Override
- public String getSourceNodeTargetTypeCfName() {
- return "Graph_Source_Node_Target_Type";
- }
-
-
- @Override
- public String getTargetNodeSourceTypeCfName() {
- return "Graph_Target_Node_Source_Type";
- }
-
-
- @Override
- public String getGraphEdgeVersions() {
- return "Graph_Edge_Versions";
+ public void increment( final ApplicationScope scope, final Shard shard,
+ final long count, final DirectedEdgeMeta directedEdgeMeta) {
+ shardApproximation.increment( scope, shard, count, directedEdgeMeta );
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e9d652dd/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/SourceEdgeSearcher.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/SourceEdgeSearcher.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/SourceEdgeSearcher.java
new file mode 100644
index 0000000..b33fcaf
--- /dev/null
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/SourceEdgeSearcher.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.usergrid.persistence.graph.serialization.impl.shard.impl;
+
+
+public class SourceEdgeSearcher {}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e9d652dd/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/ComittedGraphManagerIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/ComittedGraphManagerIT.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/ComittedGraphManagerIT.java
deleted file mode 100644
index 596480d..0000000
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/ComittedGraphManagerIT.java
+++ /dev/null
@@ -1,138 +0,0 @@
-/*
- *
- * * Licensed to the Apache Software Foundation (ASF) under one
- * * or more contributor license agreements. See the NOTICE file
- * * distributed with this work for additional information
- * * regarding copyright ownership. The ASF licenses this file
- * * to you under the Apache License, Version 2.0 (the
- * * "License"); you may not use this file except in compliance
- * * with the License. You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing,
- * * software distributed under the License is distributed on an
- * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * * KIND, either express or implied. See the License for the
- * * specific language governing permissions and limitations
- * * under the License.
- *
- */
-package org.apache.usergrid.persistence.graph;
-
-
-import java.util.concurrent.TimeUnit;
-
-import org.jukito.JukitoRunner;
-import org.jukito.UseModules;
-import org.junit.runner.RunWith;
-
-import org.apache.usergrid.persistence.core.cassandra.ITRunner;
-import org.apache.usergrid.persistence.graph.guice.TestGraphModule;
-import org.apache.usergrid.persistence.model.entity.Id;
-
-import rx.Observable;
-
-
-/**
- * Integration test that performs all calls immediately after writes without blocking. Tests that our
- * view is immediately consistent to our users, even if we have yet to perform background processing
- */
-@RunWith(ITRunner.class)
-@UseModules({ TestGraphModule.class })
-public class ComittedGraphManagerIT extends GraphManagerIT {
-
-
- @Override
- protected GraphManager getHelper(GraphManager gm) {
- return new ComittedGraphTestHelper( gm );
- }
-
-
- /**
- * Doesn't wait for the async process to happen before returning. Simply executes and immediately returns.
- */
- public static class ComittedGraphTestHelper implements GraphManager {
-
- private final GraphManager graphManager;
-
-
- public ComittedGraphTestHelper( final GraphManager graphManager ) {
- this.graphManager = graphManager;
- }
-
-
- @Override
- public Observable<Edge> writeEdge( final Edge edge ) {
- return graphManager.writeEdge( edge );
- }
-
-
- @Override
- public Observable<Edge> deleteEdge( final Edge edge ) {
- return graphManager.deleteEdge( edge );
- }
-
-
- @Override
- public Observable<Id> deleteNode( final Id node, final long timestamp) {
- return graphManager.deleteNode( node, timestamp );
- }
-
-
- @Override
- public Observable<Edge> loadEdgeVersions( final SearchByEdge edge ) {
- return graphManager.loadEdgeVersions( edge );
- }
-
-
- @Override
- public Observable<Edge> loadEdgesFromSource( final SearchByEdgeType search ) {
- return graphManager.loadEdgesFromSource( search );
- }
-
-
- @Override
- public Observable<Edge> loadEdgesToTarget( final SearchByEdgeType search ) {
- return graphManager.loadEdgesToTarget( search );
- }
-
-
- @Override
- public Observable<Edge> loadEdgesFromSourceByType( final SearchByIdType search ) {
- return graphManager.loadEdgesFromSourceByType(search);
- }
-
-
- @Override
- public Observable<Edge> loadEdgesToTargetByType( final SearchByIdType search ) {
- return graphManager.loadEdgesToTargetByType( search );
- }
-
-
- @Override
- public Observable<String> getEdgeTypesFromSource( final SearchEdgeType search ) {
- return graphManager.getEdgeTypesFromSource( search );
- }
-
-
- @Override
- public Observable<String> getIdTypesFromSource( final SearchIdType search ) {
- return graphManager.getIdTypesFromSource( search );
- }
-
-
- @Override
- public Observable<String> getEdgeTypesToTarget( final SearchEdgeType search ) {
- return graphManager.getEdgeTypesToTarget( search );
- }
-
-
- @Override
- public Observable<String> getIdTypesToTarget( final SearchIdType search ) {
- return graphManager.getIdTypesToTarget( search );
- }
-
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e9d652dd/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/CommittedGraphManagerIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/CommittedGraphManagerIT.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/CommittedGraphManagerIT.java
new file mode 100644
index 0000000..3c04ab4
--- /dev/null
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/CommittedGraphManagerIT.java
@@ -0,0 +1,138 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one
+ * * or more contributor license agreements. See the NOTICE file
+ * * distributed with this work for additional information
+ * * regarding copyright ownership. The ASF licenses this file
+ * * to you under the Apache License, Version 2.0 (the
+ * * "License"); you may not use this file except in compliance
+ * * with the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing,
+ * * software distributed under the License is distributed on an
+ * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * * KIND, either express or implied. See the License for the
+ * * specific language governing permissions and limitations
+ * * under the License.
+ *
+ */
+package org.apache.usergrid.persistence.graph;
+
+
+import java.util.concurrent.TimeUnit;
+
+import org.jukito.JukitoRunner;
+import org.jukito.UseModules;
+import org.junit.runner.RunWith;
+
+import org.apache.usergrid.persistence.core.cassandra.ITRunner;
+import org.apache.usergrid.persistence.graph.guice.TestGraphModule;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import rx.Observable;
+
+
+/**
+ * Integration test that performs all calls immediately after writes without blocking. Tests that our
+ * view is immediately consistent to our users, even if we have yet to perform background processing
+ */
+@RunWith(ITRunner.class)
+@UseModules({ TestGraphModule.class })
+public class CommittedGraphManagerIT extends GraphManagerIT {
+
+
+ @Override
+ protected GraphManager getHelper(GraphManager gm) {
+ return new ComittedGraphTestHelper( gm );
+ }
+
+
+ /**
+ * Doesn't wait for the async process to happen before returning. Simply executes and immediately returns.
+ */
+ public static class ComittedGraphTestHelper implements GraphManager {
+
+ private final GraphManager graphManager;
+
+
+ public ComittedGraphTestHelper( final GraphManager graphManager ) {
+ this.graphManager = graphManager;
+ }
+
+
+ @Override
+ public Observable<Edge> writeEdge( final Edge edge ) {
+ return graphManager.writeEdge( edge );
+ }
+
+
+ @Override
+ public Observable<Edge> deleteEdge( final Edge edge ) {
+ return graphManager.deleteEdge( edge );
+ }
+
+
+ @Override
+ public Observable<Id> deleteNode( final Id node, final long timestamp) {
+ return graphManager.deleteNode( node, timestamp );
+ }
+
+
+ @Override
+ public Observable<Edge> loadEdgeVersions( final SearchByEdge edge ) {
+ return graphManager.loadEdgeVersions( edge );
+ }
+
+
+ @Override
+ public Observable<Edge> loadEdgesFromSource( final SearchByEdgeType search ) {
+ return graphManager.loadEdgesFromSource( search );
+ }
+
+
+ @Override
+ public Observable<Edge> loadEdgesToTarget( final SearchByEdgeType search ) {
+ return graphManager.loadEdgesToTarget( search );
+ }
+
+
+ @Override
+ public Observable<Edge> loadEdgesFromSourceByType( final SearchByIdType search ) {
+ return graphManager.loadEdgesFromSourceByType(search);
+ }
+
+
+ @Override
+ public Observable<Edge> loadEdgesToTargetByType( final SearchByIdType search ) {
+ return graphManager.loadEdgesToTargetByType( search );
+ }
+
+
+ @Override
+ public Observable<String> getEdgeTypesFromSource( final SearchEdgeType search ) {
+ return graphManager.getEdgeTypesFromSource( search );
+ }
+
+
+ @Override
+ public Observable<String> getIdTypesFromSource( final SearchIdType search ) {
+ return graphManager.getIdTypesFromSource( search );
+ }
+
+
+ @Override
+ public Observable<String> getEdgeTypesToTarget( final SearchEdgeType search ) {
+ return graphManager.getEdgeTypesToTarget( search );
+ }
+
+
+ @Override
+ public Observable<String> getIdTypesToTarget( final SearchIdType search ) {
+ return graphManager.getIdTypesToTarget( search );
+ }
+
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e9d652dd/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardingIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardingIT.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardingIT.java
index 01e07b1..4fe3098 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardingIT.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardingIT.java
@@ -34,7 +34,10 @@ import org.apache.usergrid.persistence.core.cassandra.ITRunner;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
import org.apache.usergrid.persistence.graph.guice.TestGraphModule;
+import org.apache.usergrid.persistence.graph.serialization.impl.shard.DirectedEdgeMeta;
import org.apache.usergrid.persistence.graph.serialization.impl.shard.NodeShardApproximation;
+import org.apache.usergrid.persistence.graph.serialization.impl.shard.NodeType;
+import org.apache.usergrid.persistence.graph.serialization.impl.shard.Shard;
import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
@@ -101,7 +104,7 @@ public class GraphManagerShardingIT {
//each edge causes 4 counts
final long writeCount = flushCount/4;
- assertTrue( "Shard size must be >= flush Count", maxShardSize >= flushCount );
+ assertTrue( "Shard size must be >= beginFlush Count", maxShardSize >= flushCount );
Id targetId = null;
@@ -115,14 +118,21 @@ public class GraphManagerShardingIT {
}
- long shardCount = nodeShardApproximation.getCount( scope, sourceId, 0l, edgeType );
+
+ final DirectedEdgeMeta sourceEdgeMeta = DirectedEdgeMeta.fromSourceNodeTargetType(sourceId, edgeType, targetId.getType() );
+ final Shard shard = new Shard(0, 0, true);
+
+
+ long shardCount = nodeShardApproximation.getCount( scope, shard, sourceEdgeMeta );
assertEquals("Shard count for source node should be the same as write count", writeCount, shardCount);
//now verify it's correct for the target
+ final DirectedEdgeMeta targetEdgeMeta = DirectedEdgeMeta.fromTargetNodeSourceType(targetId, edgeType, sourceId.getType() );
+
- shardCount = nodeShardApproximation.getCount( scope, targetId, 0l, edgeType );
+ shardCount = nodeShardApproximation.getCount( scope, shard, targetEdgeMeta );
assertEquals(1, shardCount);
@@ -144,14 +154,10 @@ public class GraphManagerShardingIT {
final long maxShardSize = graphFig.getShardSize();
-
-
- final long startTime = System.currentTimeMillis();
-
- //each edge causes 4 counts
+ //each edge causes 4 counts
final long writeCount = flushCount/4;
- assertTrue( "Shard size must be >= flush Count", maxShardSize >= flushCount );
+ assertTrue( "Shard size must be >= beginFlush Count", maxShardSize >= flushCount );
Id sourceId = null;
@@ -165,14 +171,27 @@ public class GraphManagerShardingIT {
}
- long shardCount = nodeShardApproximation.getCount( scope, targetId, 0l, edgeType );
+ //this is from target->source, since the target id doesn't change
+ final DirectedEdgeMeta targetMeta = DirectedEdgeMeta.fromTargetNode( targetId, edgeType );
+ final Shard shard = new Shard(0l, 0l, true);
+
+ long targetWithType = nodeShardApproximation.getCount( scope, shard, targetMeta );
+
+ assertEquals("Shard count for target node should be the same as write count", writeCount, targetWithType);
- assertEquals("Shard count for source node should be the same as write count", writeCount, shardCount);
+
+ final DirectedEdgeMeta targetNodeSource = DirectedEdgeMeta.fromTargetNodeSourceType( targetId, edgeType, "source" );
+
+ long shardCount = nodeShardApproximation.getCount( scope, shard, targetNodeSource );
+
+ assertEquals("Shard count for target node should be the same as write count", writeCount, shardCount);
//now verify it's correct for the target
- shardCount = nodeShardApproximation.getCount( scope, sourceId, 0l, edgeType );
+ final DirectedEdgeMeta sourceMeta = DirectedEdgeMeta.fromSourceNode( sourceId, edgeType );
+
+ shardCount = nodeShardApproximation.getCount( scope, shard, sourceMeta );
assertEquals(1, shardCount);
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e9d652dd/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/EdgeShardSerializationTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/EdgeShardSerializationTest.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/EdgeShardSerializationTest.java
index 479e1bf..da57b0a 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/EdgeShardSerializationTest.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/EdgeShardSerializationTest.java
@@ -29,10 +29,10 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.RunWith;
+import org.apache.usergrid.persistence.collection.guice.MigrationManagerRule;
+import org.apache.usergrid.persistence.core.cassandra.CassandraRule;
import org.apache.usergrid.persistence.core.cassandra.ITRunner;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.core.cassandra.CassandraRule;
-import org.apache.usergrid.persistence.collection.guice.MigrationManagerRule;
import org.apache.usergrid.persistence.graph.guice.TestGraphModule;
import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
@@ -49,8 +49,8 @@ import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
-@RunWith( ITRunner.class )
-@UseModules( { TestGraphModule.class } )
+@RunWith(ITRunner.class)
+@UseModules({ TestGraphModule.class })
public class EdgeShardSerializationTest {
@ClassRule
@@ -86,39 +86,54 @@ public class EdgeShardSerializationTest {
final Id now = createId( "test" );
- final long slice1 = 1000l;
+ final long timestamp = 10000l;
+
+ final Shard shard1 = new Shard( 1000l, timestamp, false );
- final long slice2 = slice1 * 2;
+ final Shard shard2 = new Shard( shard1.getShardIndex() * 2, timestamp, true );
- final long slice3 = slice2 * 2;
+ final Shard shard3 = new Shard( shard2.getShardIndex() * 2, timestamp, false );
- String[] types = { "edgeType", "subType" };
+ final DirectedEdgeMeta sourceEdgeMeta = DirectedEdgeMeta.fromSourceNodeTargetType(now, "edgeType", "subType" );
- MutationBatch batch = edgeShardSerialization.writeEdgeMeta( scope, now, slice1, types );
+ MutationBatch batch = edgeShardSerialization.writeShardMeta( scope, shard1, sourceEdgeMeta );
- batch.mergeShallow( edgeShardSerialization.writeEdgeMeta( scope, now, slice2, types ) );
+ batch.mergeShallow( edgeShardSerialization.writeShardMeta( scope, shard2, sourceEdgeMeta ) );
- batch.mergeShallow( edgeShardSerialization.writeEdgeMeta( scope, now, slice3, types ) );
+ batch.mergeShallow( edgeShardSerialization.writeShardMeta( scope, shard3, sourceEdgeMeta ) );
batch.execute();
- Iterator<Long> results = edgeShardSerialization.getEdgeMetaData( scope, now, Optional.<Long>absent(), types );
+ Iterator<Shard> results =
+ edgeShardSerialization.getShardMetaData( scope, Optional.<Shard>absent(), sourceEdgeMeta );
+
- assertEquals( slice3, results.next().longValue() );
+ assertEquals( shard3, results.next() );
- assertEquals( slice2, results.next().longValue() );
+ assertEquals( shard2, results.next() );
- assertEquals( slice1, results.next().longValue() );
+ assertEquals( shard1, results.next() );
+
+
+ assertFalse( results.hasNext() );
+
+ final DirectedEdgeMeta targetEdgeMeta = DirectedEdgeMeta.fromTargetNodeSourceType( now, "edgeType", "subType" );
+
+ //test we get nothing with the other node type
+ results =
+ edgeShardSerialization.getShardMetaData( scope, Optional.<Shard>absent(), targetEdgeMeta );
assertFalse( results.hasNext() );
+
//test paging and size
- results = edgeShardSerialization.getEdgeMetaData( scope, now, Optional.of( slice2 ), types );
+ results = edgeShardSerialization.getShardMetaData( scope, Optional.of( shard2 ), sourceEdgeMeta );
- assertEquals( slice2, results.next().longValue() );
+ assertEquals( shard2, results.next() );
- assertEquals( slice1, results.next().longValue() );
+
+ assertEquals( shard1, results.next() );
assertFalse( results.hasNext() );
@@ -130,50 +145,71 @@ public class EdgeShardSerializationTest {
final Id now = createId( "test" );
- final long slice1 = 1000l;
- final long slice2 = slice1 * 2;
+ final long timestamp = 10000l;
+
+ final Shard shard1 = new Shard( 1000l, timestamp, false );
+
+ final Shard shard2 = new Shard( shard1.getShardIndex() * 2, timestamp, true );
- final long slice3 = slice2 * 2;
+ final Shard shard3 = new Shard( shard2.getShardIndex() * 2, timestamp, false );
- String[] types = { "edgeType", "subType" };
- MutationBatch batch = edgeShardSerialization.writeEdgeMeta( scope, now, slice1, types );
+ final DirectedEdgeMeta sourceEdgeMeta = DirectedEdgeMeta.fromSourceNodeTargetType(now, "edgeType", "subType" );
- batch.mergeShallow( edgeShardSerialization.writeEdgeMeta( scope, now, slice2, types ) );
- batch.mergeShallow( edgeShardSerialization.writeEdgeMeta( scope, now, slice3, types ) );
+ MutationBatch batch =
+ edgeShardSerialization.writeShardMeta( scope, shard1, sourceEdgeMeta );
+
+ batch.mergeShallow(
+ edgeShardSerialization.writeShardMeta( scope, shard2, sourceEdgeMeta ) );
+
+ batch.mergeShallow(
+ edgeShardSerialization.writeShardMeta( scope, shard3, sourceEdgeMeta ) );
batch.execute();
- Iterator<Long> results = edgeShardSerialization.getEdgeMetaData( scope, now, Optional.<Long>absent(), types );
+ Iterator<Shard> results =
+ edgeShardSerialization.getShardMetaData( scope, Optional.<Shard>absent(), sourceEdgeMeta );
- assertEquals( slice3, results.next().longValue() );
+ assertEquals( shard3, results.next() );
- assertEquals( slice2, results.next().longValue() );
+ assertEquals( shard2, results.next() );
- assertEquals( slice1, results.next().longValue() );
+ assertEquals( shard1, results.next() );
assertFalse( results.hasNext() );
+ //test nothing with other type
+
+ final DirectedEdgeMeta targetEdgeMeta = DirectedEdgeMeta.fromTargetNodeSourceType( now, "edgeType", "subType" );
+
+ results =
+ edgeShardSerialization.getShardMetaData( scope, Optional.<Shard>absent(), targetEdgeMeta );
+
+ assertFalse( results.hasNext() );
+
+
//test paging and size
- edgeShardSerialization.removeEdgeMeta( scope, now, slice1, types ).execute();
+ edgeShardSerialization.removeShardMeta( scope, shard1, sourceEdgeMeta ).execute();
- results = edgeShardSerialization.getEdgeMetaData( scope, now,Optional.<Long>absent(), types );
+ results =
+ edgeShardSerialization.getShardMetaData( scope, Optional.<Shard>absent(), sourceEdgeMeta );
- assertEquals( slice3, results.next().longValue() );
+ assertEquals( shard3, results.next() );
- assertEquals( slice2, results.next().longValue() );
+ assertEquals( shard2, results.next() );
assertFalse( results.hasNext() );
- edgeShardSerialization.removeEdgeMeta( scope, now, slice2, types ).execute();
+ edgeShardSerialization.removeShardMeta( scope, shard2, sourceEdgeMeta ).execute();
- edgeShardSerialization.removeEdgeMeta( scope, now, slice3, types ).execute();
+ edgeShardSerialization.removeShardMeta( scope, shard3, sourceEdgeMeta ).execute();
- results = edgeShardSerialization.getEdgeMetaData( scope, now, Optional.<Long>absent(), types );
+ results =
+ edgeShardSerialization.getShardMetaData( scope, Optional.<Shard>absent(), sourceEdgeMeta );
assertFalse( results.hasNext() );