You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sn...@apache.org on 2014/02/20 18:20:42 UTC
[3/5] git commit: Initial commit with node verification. Need to
collapse into buffer for round trip optimization.
Initial commit with node verification. Need to collapse into buffer for round trip optimization.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/8c63bf45
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/8c63bf45
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/8c63bf45
Branch: refs/heads/two-dot-o
Commit: 8c63bf451f7b164a39325a474ceee37738c6c167
Parents: 62f2bad
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Feb 19 15:42:46 2014 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Feb 19 15:42:46 2014 -0700
----------------------------------------------------------------------
.../usergrid/persistence/graph/GraphFig.java | 2 +
.../persistence/graph/impl/EdgeManagerImpl.java | 83 ++++++++++++++------
.../impl/parse/ObservableIterator.java | 2 +
3 files changed, 65 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8c63bf45/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/GraphFig.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/GraphFig.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/GraphFig.java
index 0718f51..c5a2d5a 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/GraphFig.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/GraphFig.java
@@ -30,5 +30,7 @@ public interface GraphFig extends GuicyFig {
@Default( "CL_QUORUM" )
@Key( WRITE_CL )
String getWriteCL();
+
+
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8c63bf45/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/EdgeManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/EdgeManagerImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/EdgeManagerImpl.java
index db269a1..3e73edd 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/EdgeManagerImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/EdgeManagerImpl.java
@@ -21,12 +21,15 @@ package org.apache.usergrid.persistence.graph.impl;
import java.util.Iterator;
+import java.util.List;
import java.util.UUID;
+import java.util.concurrent.ExecutionException;
import org.apache.usergrid.persistence.collection.OrganizationScope;
import org.apache.usergrid.persistence.collection.mvcc.entity.ValidationUtils;
import org.apache.usergrid.persistence.graph.Edge;
import org.apache.usergrid.persistence.graph.EdgeManager;
+import org.apache.usergrid.persistence.graph.GraphFig;
import org.apache.usergrid.persistence.graph.MarkedEdge;
import org.apache.usergrid.persistence.graph.SearchByEdgeType;
import org.apache.usergrid.persistence.graph.SearchByIdType;
@@ -41,6 +44,10 @@ import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
import com.fasterxml.uuid.UUIDComparator;
+import com.google.common.base.Optional;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
import com.google.inject.Inject;
import com.google.inject.assistedinject.Assisted;
import com.netflix.astyanax.MutationBatch;
@@ -69,14 +76,18 @@ public class EdgeManagerImpl implements EdgeManager {
private final NodeSerialization nodeSerialization;
+ private final GraphFig graphFig;
+
@Inject
public EdgeManagerImpl( final Scheduler scheduler, final EdgeMetadataSerialization edgeMetadataSerialization,
- final EdgeSerialization edgeSerialization, final NodeSerialization nodeSerialization, @Assisted final OrganizationScope scope ) {
+ final EdgeSerialization edgeSerialization, final NodeSerialization nodeSerialization,
+ final GraphFig graphFig, @Assisted final OrganizationScope scope ) {
this.scheduler = scheduler;
this.edgeMetadataSerialization = edgeMetadataSerialization;
this.edgeSerialization = edgeSerialization;
this.nodeSerialization = nodeSerialization;
+ this.graphFig = graphFig;
ValidationUtils.validateOrganizationScope( scope );
@@ -145,13 +156,13 @@ public class EdgeManagerImpl implements EdgeManager {
//mark the node as deleted
final UUID deleteTime = UUIDGenerator.newTimeUUID();
- final MutationBatch nodeMutation = nodeSerialization.mark( scope, id, deleteTime);
+ final MutationBatch nodeMutation = nodeSerialization.mark( scope, id, deleteTime );
try {
nodeMutation.execute();
}
catch ( ConnectionException e ) {
- throw new RuntimeException( "Unable to connect to cassandra", e);
+ throw new RuntimeException( "Unable to connect to cassandra", e );
}
return id;
@@ -167,7 +178,15 @@ public class EdgeManagerImpl implements EdgeManager {
protected Iterator<MarkedEdge> getIterator() {
return edgeSerialization.getEdgesFromSource( scope, search );
}
- } ).filter( new EdgeFilter( search.getMaxVersion() ) )
+ } ).buffer( graphFig.getScanPageSize() ).flatMap( new Func1<List<MarkedEdge>, Observable<MarkedEdge>>() {
+ @Override
+ public Observable<MarkedEdge> call( final List<MarkedEdge> markedEdges ) {
+ return Observable.from( markedEdges ).filter( new EdgeFilter( search.getMaxVersion() ) );
+ }
+ } )
+
+
+ // .filter( new EdgeFilter( search.getMaxVersion() ) )
//we intentionally use distinct until changed. This way we won't store all the keys since this
//would hog far too much ram.
.distinctUntilChanged( new Func1<Edge, Id>() {
@@ -175,7 +194,7 @@ public class EdgeManagerImpl implements EdgeManager {
public Id call( final Edge edge ) {
return edge.getTargetNode();
}
- } ).map( mapper );
+ } ).cast( Edge.class );
}
@@ -194,7 +213,7 @@ public class EdgeManagerImpl implements EdgeManager {
public Id call( final Edge edge ) {
return edge.getSourceNode();
}
- } ).map( mapper );
+ } ).cast( Edge.class );
}
@@ -205,7 +224,7 @@ public class EdgeManagerImpl implements EdgeManager {
protected Iterator<MarkedEdge> getIterator() {
return edgeSerialization.getEdgesFromSourceByTargetType( scope, search );
}
- } ).filter( new EdgeFilter( search.getMaxVersion() ) ).takeFirst().map( mapper );
+ } ).filter( new EdgeFilter( search.getMaxVersion() ) ).takeFirst().cast( Edge.class );
}
@@ -216,7 +235,7 @@ public class EdgeManagerImpl implements EdgeManager {
protected Iterator<MarkedEdge> getIterator() {
return edgeSerialization.getEdgesToTargetBySourceType( scope, search );
}
- } ).filter( new EdgeFilter( search.getMaxVersion() ) ).takeFirst().map( mapper );
+ } ).filter( new EdgeFilter( search.getMaxVersion() ) ).takeFirst().cast( Edge.class );
}
@@ -269,9 +288,18 @@ public class EdgeManagerImpl implements EdgeManager {
/**
* Filter the returned values based on the max uuid and if it's been marked for deletion or not
*/
- private static class EdgeFilter implements Func1<MarkedEdge, Boolean> {
+ private class EdgeFilter implements Func1<MarkedEdge, Boolean> {
private final UUID maxVersion;
+ private final LoadingCache<Id, Optional<UUID>> markCache =
+ CacheBuilder.newBuilder().maximumSize( graphFig.getScanPageSize()*2 ).build( new CacheLoader<Id, Optional<UUID>>() {
+
+
+ @Override
+ public Optional<UUID> load( final Id key ) throws Exception {
+ return nodeSerialization.getMaxVersion( scope, key );
+ }
+ } );
private EdgeFilter( final UUID maxVersion ) {
@@ -281,25 +309,36 @@ public class EdgeManagerImpl implements EdgeManager {
@Override
public Boolean call( final MarkedEdge edge ) {
- //our edge needs to not be deleted and have a version that's <= max Version
- return !edge.isDeleted() && UUIDComparator.staticCompare( edge.getVersion(), maxVersion ) < 1;
- }
- }
-
+ final UUID edgeVersion = edge.getVersion();
+ //our edge needs to not be deleted and have a version that's > max Version
+ if ( edge.isDeleted() || UUIDComparator.staticCompare( edgeVersion, maxVersion ) > 0 ) {
+ return false;
+ }
+ try {
+ final Optional<UUID> sourceId = markCache.get( edge.getSourceNode() );
+ //the source Id has been marked for deletion. It's version is <= to the marked version for deletion,
+ // so we need to discard it
+ if ( sourceId.isPresent() && UUIDComparator.staticCompare( edgeVersion, sourceId.get() ) > -1 ) {
+ return false;
+ }
+ final Optional<UUID> targetId = markCache.get( edge.getTargetNode() );
+ //the target Id has been marked for deletion. It's version is <= to the marked version for deletion,
+ // so we need to discard it
+ if ( targetId.isPresent() && UUIDComparator.staticCompare( edgeVersion, targetId.get() ) > -1 ) {
+ return false;
+ }
+ }
+ catch ( ExecutionException e ) {
+ throw new RuntimeException( "Unable to evaluate edge for filtering", e );
+ }
- /**
- * Simple function that maps MarkedEdge to edges
- */
- private static final Func1<MarkedEdge, Edge> mapper = new Func1<MarkedEdge, Edge>() {
- @Override
- public Edge call( final MarkedEdge markedEdge ) {
- return markedEdge;
+ return true;
}
- };
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8c63bf45/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/parse/ObservableIterator.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/parse/ObservableIterator.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/parse/ObservableIterator.java
index 5bbe091..58ff6c7 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/parse/ObservableIterator.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/parse/ObservableIterator.java
@@ -43,6 +43,8 @@ public abstract class ObservableIterator<T> implements Observable.OnSubscribeFun
}
+
+
/**
* Return the iterator to feed data to
*/