You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by to...@apache.org on 2015/05/10 13:00:17 UTC
[1/9] incubator-usergrid git commit: First pass at refactoring mark +
sweep
Repository: incubator-usergrid
Updated Branches:
refs/heads/two-dot-o-dev cf8b55159 -> d0bde1884
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3e2afe23/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupCompactionImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupCompactionImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupCompactionImpl.java
index 6135121..619e65d 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupCompactionImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupCompactionImpl.java
@@ -29,7 +29,9 @@ import java.util.Iterator;
import java.util.Random;
import java.util.Set;
import java.util.UUID;
+import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.RejectedExecutionException;
import javax.annotation.Nullable;
@@ -37,13 +39,11 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.usergrid.persistence.core.consistency.TimeService;
+import org.apache.usergrid.persistence.core.executor.TaskExecutorFactory;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.core.task.Task;
-import org.apache.usergrid.persistence.core.task.TaskExecutor;
import org.apache.usergrid.persistence.graph.GraphFig;
import org.apache.usergrid.persistence.graph.MarkedEdge;
import org.apache.usergrid.persistence.graph.SearchByEdgeType;
-import org.apache.usergrid.persistence.graph.guice.GraphTaskExecutor;
import org.apache.usergrid.persistence.graph.serialization.impl.shard.DirectedEdgeMeta;
import org.apache.usergrid.persistence.graph.serialization.impl.shard.EdgeColumnFamilies;
import org.apache.usergrid.persistence.graph.serialization.impl.shard.EdgeShardSerialization;
@@ -63,6 +63,8 @@ import com.google.common.hash.PrimitiveSink;
import com.google.common.util.concurrent.FutureCallback;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.ListeningExecutorService;
+import com.google.common.util.concurrent.MoreExecutors;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import com.netflix.astyanax.Keyspace;
@@ -85,7 +87,7 @@ public class ShardGroupCompactionImpl implements ShardGroupCompaction {
private static final HashFunction MURMUR_128 = Hashing.murmur3_128();
- private final TaskExecutor taskExecutor;
+ private final ListeningExecutorService taskExecutor;
private final TimeService timeService;
private final GraphFig graphFig;
private final NodeShardAllocation nodeShardAllocation;
@@ -104,8 +106,7 @@ public class ShardGroupCompactionImpl implements ShardGroupCompaction {
final NodeShardAllocation nodeShardAllocation,
final ShardedEdgeSerialization shardedEdgeSerialization,
final EdgeColumnFamilies edgeColumnFamilies, final Keyspace keyspace,
- final EdgeShardSerialization edgeShardSerialization,
- @GraphTaskExecutor final TaskExecutor taskExecutor ) {
+ final EdgeShardSerialization edgeShardSerialization) {
this.timeService = timeService;
this.graphFig = graphFig;
@@ -119,7 +120,10 @@ public class ShardGroupCompactionImpl implements ShardGroupCompaction {
this.shardCompactionTaskTracker = new ShardCompactionTaskTracker();
this.shardAuditTaskTracker = new ShardAuditTaskTracker();
- this.taskExecutor = taskExecutor;
+
+ this.taskExecutor = MoreExecutors.listeningDecorator( TaskExecutorFactory
+ .createTaskExecutor( "ShardCompaction", graphFig.getShardAuditWorkerCount(),
+ graphFig.getShardAuditWorkerQueueSize() ) );
}
@@ -139,8 +143,8 @@ public class ShardGroupCompactionImpl implements ShardGroupCompaction {
Preconditions.checkNotNull( group, "group cannot be null" );
Preconditions.checkArgument( group.isCompactionPending(), "Compaction is pending" );
- Preconditions.checkArgument( group.shouldCompact( startTime ),
- "Compaction cannot be run yet. Ignoring compaction." );
+ Preconditions
+ .checkArgument( group.shouldCompact( startTime ), "Compaction cannot be run yet. Ignoring compaction." );
final CompactionResult.CompactionBuilder resultBuilder = CompactionResult.builder();
@@ -170,8 +174,9 @@ public class ShardGroupCompactionImpl implements ShardGroupCompaction {
for ( Shard sourceShard : sourceShards ) {
- Iterator<MarkedEdge> edges = edgeMeta.loadEdges( shardedEdgeSerialization, edgeColumnFamilies, scope,
- Collections.singleton( sourceShard ), Long.MAX_VALUE, SearchByEdgeType.Order.DESCENDING );
+ Iterator<MarkedEdge> edges = edgeMeta
+ .loadEdges( shardedEdgeSerialization, edgeColumnFamilies, scope, Collections.singleton( sourceShard ),
+ Long.MAX_VALUE, SearchByEdgeType.Order.DESCENDING );
while ( edges.hasNext() ) {
final MarkedEdge edge = edges.next();
@@ -186,13 +191,13 @@ public class ShardGroupCompactionImpl implements ShardGroupCompaction {
}
- newRowBatch.mergeShallow(
- edgeMeta.writeEdge( shardedEdgeSerialization, edgeColumnFamilies, scope, targetShard, edge,
- timestamp ) );
+ newRowBatch.mergeShallow( edgeMeta
+ .writeEdge( shardedEdgeSerialization, edgeColumnFamilies, scope, targetShard, edge,
+ timestamp ) );
- deleteRowBatch.mergeShallow(
- edgeMeta.deleteEdge( shardedEdgeSerialization, edgeColumnFamilies, scope, sourceShard, edge,
- timestamp ) );
+ deleteRowBatch.mergeShallow( edgeMeta
+ .deleteEdge( shardedEdgeSerialization, edgeColumnFamilies, scope, sourceShard, edge,
+ timestamp ) );
edgeCount++;
@@ -298,7 +303,18 @@ public class ShardGroupCompactionImpl implements ShardGroupCompaction {
* Try and submit. During back pressure, we may not be able to submit, that's ok. Better to drop than to
* hose the system
*/
- ListenableFuture<AuditResult> future = taskExecutor.submit( new ShardAuditTask( scope, edgeMeta, group ) );
+ final ListenableFuture<AuditResult> future;
+
+ try {
+ future = taskExecutor.submit( new ShardAuditTask( scope, edgeMeta, group ) );
+ }
+ catch ( RejectedExecutionException ree ) {
+
+ //ignore, if this happens we don't care, we're saturated, we can check later
+ LOG.error( "Rejected audit for shard of scope {} edge, meta {} and group {}", scope, edgeMeta, group );
+
+ return Futures.immediateFuture( AuditResult.NOT_CHECKED );
+ }
/**
* Log our success or failures for debugging purposes
@@ -320,7 +336,7 @@ public class ShardGroupCompactionImpl implements ShardGroupCompaction {
}
- private final class ShardAuditTask implements Task<AuditResult> {
+ private final class ShardAuditTask implements Callable<AuditResult> {
private final ApplicationScope scope;
private final DirectedEdgeMeta edgeMeta;
@@ -334,20 +350,6 @@ public class ShardGroupCompactionImpl implements ShardGroupCompaction {
this.group = group;
}
- @Override
- public void exceptionThrown( final Throwable throwable ) {
- LOG.error( "Unable to execute audit for shard of {}", throwable );
- }
-
-
- @Override
- public AuditResult rejected() {
- //ignore, if this happens we don't care, we're saturated, we can check later
- LOG.error( "Rejected audit for shard of scope {} edge, meta {} and group {}", scope, edgeMeta, group );
-
- return AuditResult.NOT_CHECKED;
- }
-
@Override
public AuditResult call() throws Exception {
@@ -401,10 +403,8 @@ public class ShardGroupCompactionImpl implements ShardGroupCompaction {
*/
try {
CompactionResult result = compact( scope, edgeMeta, group );
- LOG.info(
- "Compaction result for compaction of scope {} with edge meta data of {} and shard group " +
- "{} is {}",
- new Object[] { scope, edgeMeta, group, result } );
+ LOG.info( "Compaction result for compaction of scope {} with edge meta data of {} and shard group "
+ + "{} is {}", new Object[] { scope, edgeMeta, group, result } );
}
finally {
shardCompactionTaskTracker.complete( scope, edgeMeta, group );
@@ -418,8 +418,6 @@ public class ShardGroupCompactionImpl implements ShardGroupCompaction {
}
-
-
/**
* Inner class used to track running tasks per instance
*/
@@ -534,8 +532,6 @@ public class ShardGroupCompactionImpl implements ShardGroupCompaction {
}
-
-
public static final class CompactionResult {
public final long copiedEdges;
@@ -566,12 +562,12 @@ public class ShardGroupCompactionImpl implements ShardGroupCompaction {
@Override
public String toString() {
return "CompactionResult{" +
- "copiedEdges=" + copiedEdges +
- ", targetShard=" + targetShard +
- ", sourceShards=" + sourceShards +
- ", removedShards=" + removedShards +
- ", compactedShard=" + compactedShard +
- '}';
+ "copiedEdges=" + copiedEdges +
+ ", targetShard=" + targetShard +
+ ", sourceShards=" + sourceShards +
+ ", removedShards=" + removedShards +
+ ", compactedShard=" + compactedShard +
+ '}';
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3e2afe23/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/CommittedGraphManagerIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/CommittedGraphManagerIT.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/CommittedGraphManagerIT.java
index b471119..658f4bf 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/CommittedGraphManagerIT.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/CommittedGraphManagerIT.java
@@ -66,14 +66,14 @@ public class CommittedGraphManagerIT extends GraphManagerIT {
@Override
- public Observable<Edge> deleteEdge( final Edge edge ) {
- return graphManager.deleteEdge( edge );
+ public Observable<Edge> markEdge( final Edge edge ) {
+ return graphManager.markEdge( edge );
}
@Override
- public Observable<Id> deleteNode( final Id node, final long timestamp) {
- return graphManager.deleteNode( node, timestamp );
+ public Observable<Id> markNode( final Id node, final long timestamp ) {
+ return graphManager.markNode( node, timestamp );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3e2afe23/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerIT.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerIT.java
index 2ed4f13..340c712 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerIT.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerIT.java
@@ -584,7 +584,7 @@ public abstract class GraphManagerIT {
//now delete it
- returned = gm.deleteEdge( edge ).toBlocking().last();
+ returned = gm.markEdge( edge ).toBlocking().last();
//now test retrieval, should be null
@@ -645,7 +645,7 @@ public abstract class GraphManagerIT {
//now delete it
- gm.deleteEdge( edge ).toBlocking().last();
+ gm.markEdge( edge ).toBlocking().last();
//now test retrieval, should be null
edges = gm.loadEdgesToTarget( search );
@@ -1005,7 +1005,7 @@ public abstract class GraphManagerIT {
System.out.println( "\n\n\n\n\n\n\n\n\n\n" );
- gm.deleteEdge( edge1 ).toBlocking().last();
+ gm.markEdge( edge1 ).toBlocking().last();
System.out.println( "\n\n\n\n\n\n\n\n\n\n" );
@@ -1025,7 +1025,7 @@ public abstract class GraphManagerIT {
//now delete one of the edges
- gm.deleteEdge( edge2 ).toBlocking().last();
+ gm.markEdge( edge2 ).toBlocking().last();
System.out.println( "\n\n\n\n\n\n\n\n\n\n" );
@@ -1081,7 +1081,7 @@ public abstract class GraphManagerIT {
//now delete one of the edges
- gm.deleteEdge( edge1 ).toBlocking().last();
+ gm.markEdge( edge1 ).toBlocking().last();
edges = gm.loadEdgesToTarget( createSearchByEdge( edge1.getTargetNode(), edge1.getType(), maxVersion, null ) );
@@ -1096,7 +1096,7 @@ public abstract class GraphManagerIT {
//now delete one of the edges
- gm.deleteEdge( edge2 ).toBlocking().last();
+ gm.markEdge( edge2 ).toBlocking().last();
edges = gm.loadEdgesToTarget( createSearchByEdge( edge1.getTargetNode(), edge1.getType(), maxVersion, null ) );
@@ -1146,7 +1146,7 @@ public abstract class GraphManagerIT {
//now delete one of the edges
- gm.deleteEdge( edge1 ).toBlocking().last();
+ gm.markEdge( edge1 ).toBlocking().last();
edges = gm.loadEdgesFromSourceByType(
@@ -1171,7 +1171,7 @@ public abstract class GraphManagerIT {
//now delete one of the edges
- gm.deleteEdge( edge2 ).toBlocking().last();
+ gm.markEdge( edge2 ).toBlocking().last();
edges = gm.loadEdgesFromSourceByType(
@@ -1223,7 +1223,7 @@ public abstract class GraphManagerIT {
//now delete one of the edges
- gm.deleteEdge( edge1 ).toBlocking().last();
+ gm.markEdge( edge1 ).toBlocking().last();
edges = gm.loadEdgesToTargetByType(
@@ -1249,7 +1249,7 @@ public abstract class GraphManagerIT {
//now delete one of the edges
- gm.deleteEdge( edge2 ).toBlocking().last();
+ gm.markEdge( edge2 ).toBlocking().last();
edges = gm.loadEdgesToTargetByType(
@@ -1320,7 +1320,7 @@ public abstract class GraphManagerIT {
assertFalse( "No more edges", results.hasNext() );
//mark the source node
- gm.deleteNode( sourceId, edge2.getTimestamp() ).toBlocking().last();
+ gm.markNode( sourceId, edge2.getTimestamp() ).toBlocking().last();
//now re-read, nothing should be there since they're marked
@@ -1402,7 +1402,7 @@ public abstract class GraphManagerIT {
assertFalse( "No more edges", results.hasNext() );
//mark the source node
- gm.deleteNode( targetId, edge2.getTimestamp() ).toBlocking().last();
+ gm.markNode( targetId, edge2.getTimestamp() ).toBlocking().last();
//now re-read, nothing should be there since they're marked
@@ -1641,7 +1641,7 @@ public abstract class GraphManagerIT {
public void invalidEdgeTypesDelete( ) {
final GraphManager em = emf.createEdgeManager( scope );
- em.deleteEdge( null );
+ em.markEdge( null );
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3e2afe23/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/StorageGraphManagerIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/StorageGraphManagerIT.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/StorageGraphManagerIT.java
index 6f3d388..84825aa 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/StorageGraphManagerIT.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/StorageGraphManagerIT.java
@@ -116,16 +116,16 @@ public class StorageGraphManagerIT extends GraphManagerIT {
@Override
- public Observable<Edge> deleteEdge( final Edge edge ) {
+ public Observable<Edge> markEdge( final Edge edge ) {
waitForComplete();
- return graphManager.deleteEdge( edge );
+ return graphManager.markEdge( edge );
}
@Override
- public Observable<Id> deleteNode( final Id node, final long timestamp ) {
+ public Observable<Id> markNode( final Id node, final long timestamp ) {
waitForComplete();
- return graphManager.deleteNode( node, timestamp );
+ return graphManager.markNode( node, timestamp );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3e2afe23/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/ShardGroupCompactionTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/ShardGroupCompactionTest.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/ShardGroupCompactionTest.java
index 68b6f8f..7d4b7f6 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/ShardGroupCompactionTest.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/ShardGroupCompactionTest.java
@@ -32,14 +32,12 @@ import org.junit.Test;
import org.apache.usergrid.persistence.core.consistency.TimeService;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
-import org.apache.usergrid.persistence.core.task.TaskExecutor;
import org.apache.usergrid.persistence.core.util.IdGenerator;
import org.apache.usergrid.persistence.graph.GraphFig;
import org.apache.usergrid.persistence.graph.serialization.impl.shard.impl.ShardGroupCompactionImpl;
import com.netflix.astyanax.Keyspace;
-import static org.apache.usergrid.persistence.core.util.IdGenerator.createId;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
import static org.mockito.Mockito.mock;
@@ -79,8 +77,6 @@ public class ShardGroupCompactionTest {
final EdgeShardSerialization edgeShardSerialization = mock( EdgeShardSerialization.class );
- final TaskExecutor taskExecutor = mock( TaskExecutor.class );
-
final long delta = 10000;
final long createTime = 20000;
@@ -97,7 +93,7 @@ public class ShardGroupCompactionTest {
ShardGroupCompactionImpl compaction =
new ShardGroupCompactionImpl( timeService, graphFig, nodeShardAllocation, shardedEdgeSerialization,
- edgeColumnFamilies, keyspace, edgeShardSerialization, taskExecutor );
+ edgeColumnFamilies, keyspace, edgeShardSerialization );
DirectedEdgeMeta directedEdgeMeta = DirectedEdgeMeta.fromSourceNode( IdGenerator.createId( "source" ), "test" );
[5/9] incubator-usergrid git commit: First pass at refactoring mark +
sweep
Posted by to...@apache.org.
First pass at refactoring mark + sweep
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/3e2afe23
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/3e2afe23
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/3e2afe23
Branch: refs/heads/two-dot-o-dev
Commit: 3e2afe23bb14761b395acbe1b6577ef024b0c3dd
Parents: 36b5bad
Author: Todd Nine <tn...@apigee.com>
Authored: Tue May 5 17:30:08 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Sun May 10 04:59:16 2015 -0600
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 2 +-
.../corepersistence/CpEntityManagerFactory.java | 2 +-
.../corepersistence/CpRelationManager.java | 4 +-
.../migration/AppInfoMigrationPlugin.java | 3 +-
.../collection/EntityCollectionManager.java | 20 +-
.../cache/CachedEntityCollectionManager.java | 18 +-
.../collection/event/EntityDeleted.java | 45 --
.../collection/event/EntityVersionCreated.java | 39 -
.../collection/event/EntityVersionDeleted.java | 46 --
.../collection/guice/CollectionModule.java | 39 -
.../EntityCollectionManagerFactoryImpl.java | 31 +-
.../impl/EntityCollectionManagerImpl.java | 386 ++++------
.../collection/impl/EntityDeletedTask.java | 137 ----
.../impl/EntityVersionCleanupTask.java | 247 -------
.../impl/EntityVersionCreatedTask.java | 115 ---
.../impl/EntityVersionTaskFactory.java | 60 --
.../mvcc/stage/delete/UniqueCleanup.java | 133 ++++
.../mvcc/stage/delete/VersionCompact.java | 120 ++++
.../MvccLogEntrySerializationStrategy.java | 14 +
.../serialization/SerializationFig.java | 30 +-
.../serialization/impl/LogEntryIterator.java | 114 ---
.../impl/MinMaxLogEntryIterator.java | 121 ++++
.../MvccLogEntrySerializationProxyImpl.java | 14 +
.../MvccLogEntrySerializationStrategyImpl.java | 83 ++-
.../migration/MvccEntityDataMigrationImpl.java | 30 +-
.../collection/EntityCollectionManagerIT.java | 215 ++++--
.../impl/EntityVersionCleanupTaskTest.java | 715 -------------------
.../impl/EntityVersionCreatedTaskTest.java | 241 -------
.../mvcc/stage/delete/UniqueCleanupTest.java | 712 ++++++++++++++++++
.../mvcc/stage/delete/VersionCompactTest.java | 238 ++++++
.../impl/LogEntryIteratorTest.java | 132 ----
.../impl/MinMaxLogEntryIteratorTest.java | 131 ++++
...ccLogEntrySerializationStrategyImplTest.java | 132 +++-
.../collection/util/LogEntryMock.java | 103 +--
.../src/test/resources/log4j.properties | 1 +
.../core/executor/TaskExecutorFactory.java | 95 +++
.../persistence/core/rx/RxTaskScheduler.java | 2 -
.../core/task/NamedTaskExecutorImpl.java | 286 --------
.../usergrid/persistence/core/task/Task.java | 48 --
.../persistence/core/task/TaskExecutor.java | 41 --
.../core/task/NamedTaskExecutorImplTest.java | 271 -------
.../usergrid/persistence/graph/GraphFig.java | 26 +-
.../persistence/graph/GraphManager.java | 8 +-
.../persistence/graph/GraphManagerFactory.java | 2 +-
.../persistence/graph/SearchByEdge.java | 6 +
.../persistence/graph/SearchByEdgeType.java | 6 +
.../persistence/graph/guice/GraphModule.java | 11 -
.../graph/impl/GraphManagerImpl.java | 404 ++++-------
.../graph/impl/SimpleSearchByEdge.java | 40 +-
.../graph/impl/SimpleSearchByEdgeType.java | 29 +-
.../shard/impl/ShardGroupCompactionImpl.java | 90 ++-
.../graph/CommittedGraphManagerIT.java | 8 +-
.../persistence/graph/GraphManagerIT.java | 26 +-
.../graph/StorageGraphManagerIT.java | 8 +-
.../impl/shard/ShardGroupCompactionTest.java | 6 +-
55 files changed, 2464 insertions(+), 3422 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3e2afe23/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 460fc11..8bcc73a 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -664,7 +664,7 @@ public class CpEntityManager implements EntityManager {
//delete it asynchronously
indexService.queueEntityDelete( applicationScope, entityId );
- return ecm.delete( entityId );
+ return ecm.mark( entityId );
}
else {
return Observable.empty();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3e2afe23/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index 6c375ef..e7ad682 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -369,7 +369,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
}
final ApplicationEntityIndex aei = entityIndexFactory.createApplicationEntityIndex(applicationScope);
final GraphManager managementGraphManager = managerCache.getGraphManager(managementAppScope);
- final Observable deleteNodeGraph = managementGraphManager.deleteNode(applicationId, Long.MAX_VALUE);
+ final Observable deleteNodeGraph = managementGraphManager.markNode( applicationId, Long.MAX_VALUE );
final Observable deleteAppFromIndex = aei.deleteApplication();
return Observable.concat(copyConnections, deleteNodeGraph, deleteAppFromIndex)
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3e2afe23/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index 6adeefc..df3fa82 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -528,7 +528,7 @@ public class CpRelationManager implements RelationManager {
//run our delete
final Edge collectionToItemEdge = createCollectionEdge( cpHeadEntity.getId(), collName, memberEntity.getId() );
- gm.deleteEdge( collectionToItemEdge ).toBlocking().last();
+ gm.markEdge( collectionToItemEdge ).toBlocking().last();
/**
@@ -782,7 +782,7 @@ public class CpRelationManager implements RelationManager {
//delete all the edges
final Edge lastEdge =
- gm.loadEdgeVersions( search ).flatMap( returnedEdge -> gm.deleteEdge( returnedEdge ) ).toBlocking()
+ gm.loadEdgeVersions( search ).flatMap( returnedEdge -> gm.markEdge( returnedEdge ) ).toBlocking()
.lastOrDefault( null );
if ( lastEdge != null ) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3e2afe23/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/AppInfoMigrationPlugin.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/AppInfoMigrationPlugin.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/AppInfoMigrationPlugin.java
index 30955af..97b87b3 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/AppInfoMigrationPlugin.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/AppInfoMigrationPlugin.java
@@ -42,7 +42,6 @@ import org.apache.usergrid.utils.UUIDUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import rx.Observable;
-import rx.functions.Func1;
import java.util.HashMap;
import java.util.Map;
@@ -200,7 +199,7 @@ public class AppInfoMigrationPlugin implements MigrationPlugin {
final ApplicationScope systemAppScope = getApplicationScope(CpNamingUtils.SYSTEM_APP_ID );
final EntityCollectionManager systemCollectionManager =
entityCollectionManagerFactory.createCollectionManager( systemAppScope );
- systemCollectionManager.delete(new SimpleId(uuid, "appinfos")).toBlocking().last();
+ systemCollectionManager.mark( new SimpleId( uuid, "appinfos" ) ).toBlocking().last();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3e2afe23/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityCollectionManager.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityCollectionManager.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityCollectionManager.java
index 5a329e3..9de8f41 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityCollectionManager.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityCollectionManager.java
@@ -46,11 +46,12 @@ public interface EntityCollectionManager {
/**
- * @param entityId MarkCommit the entity as deleted
+ * @param entityId MarkCommit the entity as deleted. Will not actually remove it from cassandra. This operation will
+ * also remove all unique properties for this entity
*
* @return The observable of the id after the operation has completed
*/
- Observable<Id> delete( Id entityId );
+ Observable<Id> mark( Id entityId );
/**
* @param entityId The entity id to load.
@@ -96,6 +97,21 @@ public interface EntityCollectionManager {
*/
Observable<EntitySet> load( Collection<Id> entityIds );
+ /**
+ * Get all versions of the log entry, from Max to min
+ * @param entityId
+ * @return An observable stream of mvccLog entries
+ */
+ Observable<MvccLogEntry> getVersions(final Id entityId);
+
+ /**
+ * Delete these versions from cassandra. Must be atomic so that read log entries are only removed. Entity data
+ * and log entry will be deleted
+ * @param entries
+ * @return Any observable of all successfully compacted log entries
+ */
+ Observable<MvccLogEntry> delete( final Collection<MvccLogEntry> entries );
+
/**
* Returns health of entity data store.
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3e2afe23/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/cache/CachedEntityCollectionManager.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/cache/CachedEntityCollectionManager.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/cache/CachedEntityCollectionManager.java
index fa35580..7a04b8d 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/cache/CachedEntityCollectionManager.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/cache/CachedEntityCollectionManager.java
@@ -26,6 +26,7 @@ import java.util.concurrent.TimeUnit;
import org.apache.usergrid.persistence.collection.EntityCollectionManager;
import org.apache.usergrid.persistence.collection.EntitySet;
import org.apache.usergrid.persistence.collection.FieldSet;
+import org.apache.usergrid.persistence.collection.MvccLogEntry;
import org.apache.usergrid.persistence.collection.VersionSet;
import org.apache.usergrid.persistence.core.util.Health;
import org.apache.usergrid.persistence.model.entity.Entity;
@@ -84,8 +85,8 @@ public class CachedEntityCollectionManager implements EntityCollectionManager {
@Override
- public Observable<Id> delete( final Id entityId ) {
- return targetEntityCollectionManager.delete( entityId ).doOnNext( new Action1<Id>() {
+ public Observable<Id> mark( final Id entityId ) {
+ return targetEntityCollectionManager.mark( entityId ).doOnNext( new Action1<Id>() {
@Override
public void call( final Id id ) {
entityCache.invalidate( id );
@@ -125,6 +126,19 @@ public class CachedEntityCollectionManager implements EntityCollectionManager {
return targetEntityCollectionManager.load( entityIds );
}
+
+ @Override
+ public Observable<MvccLogEntry> getVersions( final Id entityId ) {
+ return targetEntityCollectionManager.getVersions( entityId );
+ }
+
+
+ @Override
+ public Observable<MvccLogEntry> delete( final Collection<MvccLogEntry> entries ) {
+ return targetEntityCollectionManager.delete( entries );
+ }
+
+
@Override
public Health getHealth() {
return targetEntityCollectionManager.getHealth();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3e2afe23/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/event/EntityDeleted.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/event/EntityDeleted.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/event/EntityDeleted.java
deleted file mode 100644
index 0e9b62e..0000000
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/event/EntityDeleted.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. The ASF licenses this file to You
- * under the Apache License, Version 2.0 (the "License"); you may not
- * use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License. For additional information regarding
- * copyright in this work, please see the NOTICE file in the top level
- * directory of this distribution.
- */
-package org.apache.usergrid.persistence.collection.event;
-
-
-import java.util.UUID;
-
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.model.entity.Id;
-
-
-/**
- *
- * Invoked when an entity is deleted. The delete log entry is not removed until all instances of this listener has completed.
- * If any listener fails with an exception, the entity will not be removed.
- *
- */
-public interface EntityDeleted {
-
-
- /**
- * The event fired when an entity is deleted
- *
- * @param scope The scope of the entity
- * @param entityId The id of the entity
- * @param version the entity version
- */
- public void deleted( final ApplicationScope scope, final Id entityId, final UUID version);
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3e2afe23/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/event/EntityVersionCreated.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/event/EntityVersionCreated.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/event/EntityVersionCreated.java
deleted file mode 100644
index 7f1be1a..0000000
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/event/EntityVersionCreated.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. The ASF licenses this file to You
- * under the Apache License, Version 2.0 (the "License"); you may not
- * use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License. For additional information regarding
- * copyright in this work, please see the NOTICE file in the top level
- * directory of this distribution.
- */
-package org.apache.usergrid.persistence.collection.event;
-
-
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.model.entity.Entity;
-
-
-/**
- * Invoked after a new version of an entity has been created.
- * The entity should be a complete view of the entity.
- */
-public interface EntityVersionCreated {
-
- /**
- * The new version of the entity. Note that this should be a fully merged view of the entity.
- * In the case of partial updates, the passed entity should be fully merged with it's previous
- * entries.
- * @param scope The scope of the entity
- * @param entity The fully loaded and merged entity
- */
- public void versionCreated( final ApplicationScope scope, final Entity entity );
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3e2afe23/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/event/EntityVersionDeleted.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/event/EntityVersionDeleted.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/event/EntityVersionDeleted.java
deleted file mode 100644
index 7fd8fe7..0000000
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/event/EntityVersionDeleted.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. The ASF licenses this file to You
- * under the Apache License, Version 2.0 (the "License"); you may not
- * use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License. For additional information regarding
- * copyright in this work, please see the NOTICE file in the top level
- * directory of this distribution.
- */
-package org.apache.usergrid.persistence.collection.event;
-
-
-import java.util.List;
-
-import org.apache.usergrid.persistence.collection.MvccLogEntry;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.model.entity.Id;
-
-
-/**
- *
- * Invoked when an entity version is removed. Note that this is not a deletion of the entity
- * itself, only the version itself.
- *
- */
-public interface EntityVersionDeleted {
-
- /**
- * The version specified was removed.
- *
- * @param scope The scope of the entity
- * @param entityId The entity Id that was removed
- * @param entityVersions The versions that are to be removed
- */
- public void versionDeleted(final ApplicationScope scope, final Id entityId,
- final List<MvccLogEntry> entityVersions);
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3e2afe23/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/guice/CollectionModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/guice/CollectionModule.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/guice/CollectionModule.java
index b256a44..78c7f37 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/guice/CollectionModule.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/guice/CollectionModule.java
@@ -22,27 +22,14 @@ import org.safehaus.guicyfig.GuicyFigModule;
import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
import org.apache.usergrid.persistence.collection.cache.EntityCacheFig;
-import org.apache.usergrid.persistence.collection.event.EntityDeleted;
-import org.apache.usergrid.persistence.collection.event.EntityVersionCreated;
-import org.apache.usergrid.persistence.collection.event.EntityVersionDeleted;
import org.apache.usergrid.persistence.collection.impl.EntityCollectionManagerFactoryImpl;
-import org.apache.usergrid.persistence.collection.impl.EntityVersionTaskFactory;
import org.apache.usergrid.persistence.collection.mvcc.changelog.ChangeLogGenerator;
import org.apache.usergrid.persistence.collection.mvcc.changelog.ChangeLogGeneratorImpl;
import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
-import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
import org.apache.usergrid.persistence.collection.serialization.impl.SerializationModule;
-import org.apache.usergrid.persistence.collection.serialization.impl.UniqueValueSerializationStrategyImpl;
import org.apache.usergrid.persistence.collection.service.impl.ServiceModule;
-import org.apache.usergrid.persistence.core.task.NamedTaskExecutorImpl;
-import org.apache.usergrid.persistence.core.task.TaskExecutor;
import com.google.inject.AbstractModule;
-import com.google.inject.Inject;
-import com.google.inject.Provides;
-import com.google.inject.Singleton;
-import com.google.inject.assistedinject.FactoryModuleBuilder;
-import com.google.inject.multibindings.Multibinder;
/**
@@ -61,15 +48,7 @@ public abstract class CollectionModule extends AbstractModule {
install( new SerializationModule() );
install( new ServiceModule() );
- install ( new FactoryModuleBuilder().build( EntityVersionTaskFactory.class ));
-
// users of this module can add their own implemementations
- // for more information: https://github.com/google/guice/wiki/Multibindings
-
- Multibinder.newSetBinder( binder(), EntityVersionDeleted.class );
- Multibinder.newSetBinder( binder(), EntityVersionCreated.class );
- Multibinder.newSetBinder( binder(), EntityDeleted.class );
-
// create a guice factor for getting our collection manager
bind(EntityCollectionManagerFactory.class).to(EntityCollectionManagerFactoryImpl.class);
@@ -81,24 +60,6 @@ public abstract class CollectionModule extends AbstractModule {
configureMigrationProvider();
}
-//
-// @Provides
-// @Singleton
-// @Inject
-// public WriteStart write (final MvccLogEntrySerializationStrategy logStrategy) {
-// final WriteStart writeStart = new WriteStart( logStrategy, MvccEntity.Status.COMPLETE);
-//
-// return writeStart;
-// }
-
- @Inject
- @Singleton
- @Provides
- @CollectionTaskExecutor
- public TaskExecutor collectionTaskExecutor(final SerializationFig serializationFig){
- return new NamedTaskExecutorImpl( "collectiontasks",
- serializationFig.getTaskPoolThreadSize(), serializationFig.getTaskPoolQueueSize() );
- }
/**
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3e2afe23/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerFactoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerFactoryImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerFactoryImpl.java
index 761c4b5..50a4bfc 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerFactoryImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerFactoryImpl.java
@@ -27,9 +27,10 @@ import org.apache.usergrid.persistence.collection.EntityCollectionManager;
import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
import org.apache.usergrid.persistence.collection.cache.CachedEntityCollectionManager;
import org.apache.usergrid.persistence.collection.cache.EntityCacheFig;
-import org.apache.usergrid.persistence.collection.guice.CollectionTaskExecutor;
import org.apache.usergrid.persistence.collection.mvcc.stage.delete.MarkCommit;
import org.apache.usergrid.persistence.collection.mvcc.stage.delete.MarkStart;
+import org.apache.usergrid.persistence.collection.mvcc.stage.delete.UniqueCleanup;
+import org.apache.usergrid.persistence.collection.mvcc.stage.delete.VersionCompact;
import org.apache.usergrid.persistence.collection.mvcc.stage.write.RollbackAction;
import org.apache.usergrid.persistence.collection.mvcc.stage.write.WriteCommit;
import org.apache.usergrid.persistence.collection.mvcc.stage.write.WriteOptimisticVerify;
@@ -37,11 +38,11 @@ import org.apache.usergrid.persistence.collection.mvcc.stage.write.WriteStart;
import org.apache.usergrid.persistence.collection.mvcc.stage.write.WriteUniqueVerify;
import org.apache.usergrid.persistence.collection.serialization.MvccEntitySerializationStrategy;
import org.apache.usergrid.persistence.collection.serialization.MvccLogEntrySerializationStrategy;
+import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.apache.usergrid.persistence.core.rx.RxTaskScheduler;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.core.task.TaskExecutor;
import com.google.common.base.Preconditions;
import com.google.common.cache.CacheBuilder;
@@ -67,12 +68,13 @@ public class EntityCollectionManagerFactoryImpl implements EntityCollectionManag
private final RollbackAction rollback;
private final MarkStart markStart;
private final MarkCommit markCommit;
+ private final UniqueCleanup uniqueCleanup;
+ private final VersionCompact versionCompact;
+ private final SerializationFig serializationFig;
private final MvccEntitySerializationStrategy entitySerializationStrategy;
private final UniqueValueSerializationStrategy uniqueValueSerializationStrategy;
private final MvccLogEntrySerializationStrategy mvccLogEntrySerializationStrategy;
private final Keyspace keyspace;
- private final EntityVersionTaskFactory entityVersionTaskFactory;
- private final TaskExecutor taskExecutor;
private final EntityCacheFig entityCacheFig;
private final MetricsFactory metricsFactory;
private final RxTaskScheduler rxTaskScheduler;
@@ -84,10 +86,11 @@ public class EntityCollectionManagerFactoryImpl implements EntityCollectionManag
//create the target EM that will perform logic
final EntityCollectionManager target = new EntityCollectionManagerImpl(
writeStart, writeVerifyUnique,
- writeOptimisticVerify, writeCommit, rollback, markStart, markCommit,
+ writeOptimisticVerify, writeCommit, rollback, markStart, markCommit, uniqueCleanup, versionCompact,
entitySerializationStrategy, uniqueValueSerializationStrategy,
- mvccLogEntrySerializationStrategy, keyspace,entityVersionTaskFactory, taskExecutor, scope, metricsFactory,
- rxTaskScheduler );
+ mvccLogEntrySerializationStrategy, keyspace,
+ metricsFactory, serializationFig,
+ rxTaskScheduler, scope );
final EntityCollectionManager proxy = new CachedEntityCollectionManager(entityCacheFig, target );
@@ -102,13 +105,12 @@ public class EntityCollectionManagerFactoryImpl implements EntityCollectionManag
final WriteOptimisticVerify writeOptimisticVerify,
final WriteCommit writeCommit, final RollbackAction rollback,
final MarkStart markStart, final MarkCommit markCommit,
- final MvccEntitySerializationStrategy entitySerializationStrategy,
+ final UniqueCleanup uniqueCleanup, final VersionCompact versionCompact,
+ final SerializationFig serializationFig, final
+ MvccEntitySerializationStrategy entitySerializationStrategy,
final UniqueValueSerializationStrategy uniqueValueSerializationStrategy,
final MvccLogEntrySerializationStrategy mvccLogEntrySerializationStrategy,
- final Keyspace keyspace,
- final EntityVersionTaskFactory entityVersionTaskFactory,
- @CollectionTaskExecutor final TaskExecutor taskExecutor, final
- EntityCacheFig entityCacheFig,
+ final Keyspace keyspace, final EntityCacheFig entityCacheFig,
MetricsFactory metricsFactory, final RxTaskScheduler rxTaskScheduler ) {
this.writeStart = writeStart;
@@ -118,12 +120,13 @@ public class EntityCollectionManagerFactoryImpl implements EntityCollectionManag
this.rollback = rollback;
this.markStart = markStart;
this.markCommit = markCommit;
+ this.uniqueCleanup = uniqueCleanup;
+ this.versionCompact = versionCompact;
+ this.serializationFig = serializationFig;
this.entitySerializationStrategy = entitySerializationStrategy;
this.uniqueValueSerializationStrategy = uniqueValueSerializationStrategy;
this.mvccLogEntrySerializationStrategy = mvccLogEntrySerializationStrategy;
this.keyspace = keyspace;
- this.entityVersionTaskFactory = entityVersionTaskFactory;
- this.taskExecutor = taskExecutor;
this.entityCacheFig = entityCacheFig;
this.metricsFactory = metricsFactory;
this.rxTaskScheduler = rxTaskScheduler;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3e2afe23/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
index 6f10e86..7a32d72 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
@@ -22,6 +22,7 @@ package org.apache.usergrid.persistence.collection.impl;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
+import java.util.Iterator;
import java.util.List;
import java.util.UUID;
@@ -32,11 +33,13 @@ import org.apache.usergrid.persistence.collection.EntityCollectionManager;
import org.apache.usergrid.persistence.collection.EntitySet;
import org.apache.usergrid.persistence.collection.FieldSet;
import org.apache.usergrid.persistence.collection.MvccEntity;
+import org.apache.usergrid.persistence.collection.MvccLogEntry;
import org.apache.usergrid.persistence.collection.VersionSet;
-import org.apache.usergrid.persistence.collection.guice.CollectionTaskExecutor;
import org.apache.usergrid.persistence.collection.mvcc.stage.CollectionIoEvent;
import org.apache.usergrid.persistence.collection.mvcc.stage.delete.MarkCommit;
import org.apache.usergrid.persistence.collection.mvcc.stage.delete.MarkStart;
+import org.apache.usergrid.persistence.collection.mvcc.stage.delete.UniqueCleanup;
+import org.apache.usergrid.persistence.collection.mvcc.stage.delete.VersionCompact;
import org.apache.usergrid.persistence.collection.mvcc.stage.write.RollbackAction;
import org.apache.usergrid.persistence.collection.mvcc.stage.write.WriteCommit;
import org.apache.usergrid.persistence.collection.mvcc.stage.write.WriteOptimisticVerify;
@@ -44,15 +47,17 @@ import org.apache.usergrid.persistence.collection.mvcc.stage.write.WriteStart;
import org.apache.usergrid.persistence.collection.mvcc.stage.write.WriteUniqueVerify;
import org.apache.usergrid.persistence.collection.serialization.MvccEntitySerializationStrategy;
import org.apache.usergrid.persistence.collection.serialization.MvccLogEntrySerializationStrategy;
+import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
import org.apache.usergrid.persistence.collection.serialization.UniqueValue;
import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
import org.apache.usergrid.persistence.collection.serialization.UniqueValueSet;
+import org.apache.usergrid.persistence.collection.serialization.impl.MinMaxLogEntryIterator;
import org.apache.usergrid.persistence.collection.serialization.impl.MutableFieldSet;
import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
+import org.apache.usergrid.persistence.core.metrics.ObservableTimer;
+import org.apache.usergrid.persistence.core.rx.ObservableIterator;
import org.apache.usergrid.persistence.core.rx.RxTaskScheduler;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.core.task.Task;
-import org.apache.usergrid.persistence.core.task.TaskExecutor;
import org.apache.usergrid.persistence.core.util.Health;
import org.apache.usergrid.persistence.core.util.ValidationUtils;
import org.apache.usergrid.persistence.model.entity.Entity;
@@ -60,7 +65,6 @@ import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.field.Field;
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
-import com.codahale.metrics.Meter;
import com.codahale.metrics.Timer;
import com.google.common.base.Preconditions;
import com.google.inject.Inject;
@@ -73,13 +77,9 @@ import com.netflix.astyanax.model.ColumnFamily;
import com.netflix.astyanax.model.CqlResult;
import com.netflix.astyanax.serializers.StringSerializer;
-import rx.Notification;
import rx.Observable;
import rx.Subscriber;
import rx.functions.Action0;
-import rx.functions.Action1;
-import rx.functions.Func1;
-import rx.schedulers.Schedulers;
/**
@@ -97,6 +97,8 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
private final WriteOptimisticVerify writeOptimisticVerify;
private final WriteCommit writeCommit;
private final RollbackAction rollback;
+ private final UniqueCleanup uniqueCleanup;
+ private final VersionCompact versionCompact;
//delete stages
@@ -107,41 +109,39 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
private final MvccEntitySerializationStrategy entitySerializationStrategy;
private final UniqueValueSerializationStrategy uniqueValueSerializationStrategy;
- private final EntityVersionTaskFactory entityVersionTaskFactory;
- private final TaskExecutor taskExecutor;
+ private final SerializationFig serializationFig;
- private final RxTaskScheduler rxTaskScheduler;
private final Keyspace keyspace;
private final Timer writeTimer;
- private final Meter writeMeter;
private final Timer deleteTimer;
+ private final Timer fieldIdTimer;
+ private final Timer fieldEntityTimer;
private final Timer updateTimer;
private final Timer loadTimer;
private final Timer getLatestTimer;
- private final Meter deleteMeter;
- private final Meter getLatestMeter;
- private final Meter loadMeter;
- private final Meter updateMeter;
private final ApplicationScope applicationScope;
+ private final RxTaskScheduler rxTaskScheduler;
@Inject
public EntityCollectionManagerImpl( final WriteStart writeStart, final WriteUniqueVerify writeVerifyUnique,
- final WriteOptimisticVerify writeOptimisticVerify, final WriteCommit
- writeCommit,
- final RollbackAction rollback, final MarkStart markStart,
- final MarkCommit markCommit, final MvccEntitySerializationStrategy entitySerializationStrategy,
+ final WriteOptimisticVerify writeOptimisticVerify,
+ final WriteCommit writeCommit, final RollbackAction rollback,
+ final MarkStart markStart, final MarkCommit markCommit,
+ final UniqueCleanup uniqueCleanup, final VersionCompact versionCompact,
+ final MvccEntitySerializationStrategy entitySerializationStrategy,
final UniqueValueSerializationStrategy uniqueValueSerializationStrategy,
final MvccLogEntrySerializationStrategy mvccLogEntrySerializationStrategy,
- final Keyspace keyspace, final EntityVersionTaskFactory entityVersionTaskFactory,
- @CollectionTaskExecutor final TaskExecutor taskExecutor, @Assisted final ApplicationScope applicationScope,
- final MetricsFactory metricsFactory,
-
- final RxTaskScheduler rxTaskScheduler ) {
+ final Keyspace keyspace, final MetricsFactory metricsFactory,
+ final SerializationFig serializationFig, final RxTaskScheduler rxTaskScheduler,
+ @Assisted final ApplicationScope applicationScope ) {
this.uniqueValueSerializationStrategy = uniqueValueSerializationStrategy;
this.entitySerializationStrategy = entitySerializationStrategy;
+ this.uniqueCleanup = uniqueCleanup;
+ this.versionCompact = versionCompact;
+ this.serializationFig = serializationFig;
this.rxTaskScheduler = rxTaskScheduler;
ValidationUtils.validateApplicationScope( applicationScope );
@@ -158,21 +158,16 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
this.keyspace = keyspace;
- this.entityVersionTaskFactory = entityVersionTaskFactory;
- this.taskExecutor = taskExecutor;
this.applicationScope = applicationScope;
this.mvccLogEntrySerializationStrategy = mvccLogEntrySerializationStrategy;
- this.writeTimer = metricsFactory.getTimer(EntityCollectionManagerImpl.class,"write.timer");
- this.writeMeter = metricsFactory.getMeter(EntityCollectionManagerImpl.class, "write.meter");
- this.deleteTimer = metricsFactory.getTimer(EntityCollectionManagerImpl.class, "delete.timer");
- this.deleteMeter= metricsFactory.getMeter(EntityCollectionManagerImpl.class, "delete.meter");
- this.updateTimer = metricsFactory.getTimer(EntityCollectionManagerImpl.class, "update.timer");
- this.updateMeter = metricsFactory.getMeter(EntityCollectionManagerImpl.class, "update.meter");
- this.loadTimer = metricsFactory.getTimer(EntityCollectionManagerImpl.class,"load.timer");
- this.loadMeter = metricsFactory.getMeter(EntityCollectionManagerImpl.class, "load.meter");
- this.getLatestTimer = metricsFactory.getTimer(EntityCollectionManagerImpl.class,"latest.timer");
- this.getLatestMeter = metricsFactory.getMeter(EntityCollectionManagerImpl.class, "latest.meter");
+ this.writeTimer = metricsFactory.getTimer( EntityCollectionManagerImpl.class, "write" );
+ this.deleteTimer = metricsFactory.getTimer( EntityCollectionManagerImpl.class, "delete" );
+ this.fieldIdTimer = metricsFactory.getTimer( EntityCollectionManagerImpl.class, "fieldId" );
+ this.fieldEntityTimer = metricsFactory.getTimer( EntityCollectionManagerImpl.class, "fieldEntity" );
+ this.updateTimer = metricsFactory.getTimer( EntityCollectionManagerImpl.class, "update" );
+ this.loadTimer = metricsFactory.getTimer( EntityCollectionManagerImpl.class, "load" );
+ this.getLatestTimer = metricsFactory.getTimer( EntityCollectionManagerImpl.class, "latest" );
}
@@ -192,74 +187,26 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
Observable<CollectionIoEvent<MvccEntity>> observable = stageRunner( writeData, writeStart );
- // execute all validation stages concurrently. Needs refactored when this is done.
- // https://github.com/Netflix/RxJava/issues/627
- // observable = Concurrent.concurrent( observable, Schedulers.io(), new WaitZip(),
- // writeVerifyUnique, writeOptimisticVerify );
- final Timer.Context timer = writeTimer.time();
- return observable.map(writeCommit).doOnNext(new Action1<Entity>() {
- @Override
- public void call(final Entity entity) {
- //TODO fire the created task first then the entityVersioncleanup
- taskExecutor.submit( entityVersionTaskFactory.getCreatedTask( applicationScope, entity ));
- taskExecutor.submit( entityVersionTaskFactory.getCleanupTask( applicationScope, entityId,
- entity.getVersion(), false ));
- //post-processing to come later. leave it empty for now.
- }
- }).doOnError( rollback )
- .doOnEach( new Action1<Notification<? super Entity>>() {
- @Override
- public void call( Notification<? super Entity> notification ) {
- writeMeter.mark();
- }
- } )
- .doOnCompleted( new Action0() {
- @Override
- public void call() {
- timer.stop();
- }
- } );
+ final Observable<Entity> write = observable.map( writeCommit );
+
+ return ObservableTimer.time( write, writeTimer );
}
@Override
- public Observable<Id> delete( final Id entityId ) {
+ public Observable<Id> mark( final Id entityId ) {
Preconditions.checkNotNull( entityId, "Entity id is required in this stage" );
Preconditions.checkNotNull( entityId.getUuid(), "Entity id is required in this stage" );
Preconditions.checkNotNull( entityId.getType(), "Entity type is required in this stage" );
- final Timer.Context timer = deleteTimer.time();
- Observable<Id> o = Observable.just( new CollectionIoEvent<Id>( applicationScope, entityId ) )
- .map(markStart)
- .doOnNext( markCommit )
- .map(new Func1<CollectionIoEvent<MvccEntity>, Id>() {
-
- @Override
- public Id call(final CollectionIoEvent<MvccEntity> mvccEntityCollectionIoEvent) {
- MvccEntity entity = mvccEntityCollectionIoEvent.getEvent();
- Task<Void> task = entityVersionTaskFactory
- .getDeleteTask( applicationScope, entity.getId(), entity.getVersion() );
- taskExecutor.submit(task);
- return entity.getId();
- }
- }
- )
- .doOnNext(new Action1<Id>() {
- @Override
- public void call(Id id) {
- deleteMeter.mark();
- }
- })
- .doOnCompleted( new Action0() {
- @Override
- public void call() {
- timer.stop();
- }
- } );
+ Observable<Id> o = Observable.just( new CollectionIoEvent<>( applicationScope, entityId ) ).map( markStart )
+ .doOnNext( markCommit ).compose( uniqueCleanup ).map(
+ entityEvent -> entityEvent.getEvent().getId() );
+
- return o;
+ return ObservableTimer.time( o, deleteTimer );
}
@@ -270,35 +217,19 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
Preconditions.checkNotNull( entityId.getUuid(), "Entity id uuid required in load stage" );
Preconditions.checkNotNull( entityId.getType(), "Entity id type required in load stage" );
- final Timer.Context timer = loadTimer.time();
- return load( Collections.singleton( entityId ) ).flatMap(new Func1<EntitySet, Observable<Entity>>() {
- @Override
- public Observable<Entity> call(final EntitySet entitySet) {
- final MvccEntity entity = entitySet.getEntity(entityId);
-
- if (entity == null || !entity.getEntity().isPresent()) {
- return Observable.empty();
- }
+ final Observable<Entity> entityObservable = load( Collections.singleton( entityId ) ).flatMap( entitySet -> {
+ final MvccEntity entity = entitySet.getEntity( entityId );
- return Observable.just( entity.getEntity().get() );
+ if ( entity == null || !entity.getEntity().isPresent() ) {
+ return Observable.empty();
}
- })
- .doOnNext( new Action1<Entity>() {
- @Override
- public void call( Entity entity ) {
- loadMeter.mark();
- }
- } )
- .doOnCompleted( new Action0() {
- @Override
- public void call() {
- timer.stop();
- }
- } );
- }
+ return Observable.just( entity.getEntity().get() );
+ } );
+ return ObservableTimer.time( entityObservable, loadTimer );
+ }
@Override
@@ -306,15 +237,13 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
Preconditions.checkNotNull( entityIds, "entityIds cannot be null" );
- final Timer.Context timer = loadTimer.time();
-
- return Observable.create( new Observable.OnSubscribe<EntitySet>() {
+ final Observable<EntitySet> entitySetObservable = Observable.create( new Observable.OnSubscribe<EntitySet>() {
@Override
public void call( final Subscriber<? super EntitySet> subscriber ) {
try {
final EntitySet results =
- entitySerializationStrategy.load( applicationScope, entityIds, UUIDGenerator.newTimeUUID() );
+ entitySerializationStrategy.load( applicationScope, entityIds, UUIDGenerator.newTimeUUID() );
subscriber.onNext( results );
subscriber.onCompleted();
@@ -323,162 +252,167 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
subscriber.onError( e );
}
}
- } )
- .doOnNext(new Action1<EntitySet>() {
- @Override
- public void call(EntitySet entitySet) {
- loadMeter.mark();
- }
- })
- .doOnCompleted( new Action0() {
- @Override
- public void call() {
- timer.stop();
- }
- } );
+ } );
+
+
+ return ObservableTimer.time( entitySetObservable, loadTimer );
}
@Override
- public Observable<Id> getIdField(final String type, final Field field ) {
- final List<Field> fields = Collections.singletonList( field );
- return rx.Observable.from( fields ).map( new Func1<Field, Id>() {
+ public Observable<MvccLogEntry> getVersions( final Id entityId ) {
+ ValidationUtils.verifyIdentity( entityId );
+
+ return Observable.create( new ObservableIterator<MvccLogEntry>( "Log entry iterator" ) {
@Override
- public Id call( Field field ) {
- try {
- final UniqueValueSet set = uniqueValueSerializationStrategy.load( applicationScope, type, fields );
- final UniqueValue value = set.getValue( field.getName() );
- return value == null ? null : value.getEntityId();
- }
- catch ( ConnectionException e ) {
- logger.error( "Failed to getIdField", e );
- throw new RuntimeException( e );
- }
+ protected Iterator<MvccLogEntry> getIterator() {
+ return new MinMaxLogEntryIterator( mvccLogEntrySerializationStrategy, applicationScope, entityId,
+ serializationFig.getBufferSize() );
}
} );
}
+ @Override
+ public Observable<MvccLogEntry> delete( final Collection<MvccLogEntry> entries ) {
+ Preconditions.checkNotNull( entries, "entries must not be null" );
+
+
+ return Observable.from( entries ).map( logEntry -> new CollectionIoEvent<>( applicationScope, logEntry ) )
+ .compose( versionCompact ).map( event -> event.getEvent() );
+ }
+
+
+ @Override
+ public Observable<Id> getIdField( final String type, final Field field ) {
+ final List<Field> fields = Collections.singletonList( field );
+ final Observable<Id> idObservable = Observable.from( fields ).map( field1 -> {
+ try {
+ final UniqueValueSet set = uniqueValueSerializationStrategy.load( applicationScope, type, fields );
+ final UniqueValue value = set.getValue( field1.getName() );
+ return value == null ? null : value.getEntityId();
+ }
+ catch ( ConnectionException e ) {
+ logger.error( "Failed to getIdField", e );
+ throw new RuntimeException( e );
+ }
+ } );
+
+ return ObservableTimer.time( idObservable, fieldIdTimer );
+ }
+
+
/**
* Retrieves all entities that correspond to each field given in the Collection.
- * @param fields
- * @return
*/
@Override
- public Observable<FieldSet> getEntitiesFromFields(final String type, final Collection<Field> fields ) {
- return rx.Observable.just(fields).map( new Func1<Collection<Field>, FieldSet>() {
- @Override
- public FieldSet call( Collection<Field> fields ) {
- try {
-
- final UUID startTime = UUIDGenerator.newTimeUUID();
+ public Observable<FieldSet> getEntitiesFromFields( final String type, final Collection<Field> fields ) {
+ final Observable<FieldSet> fieldSetObservable = Observable.just( fields ).map( fields1 -> {
+ try {
- //Get back set of unique values that correspond to collection of fields
- UniqueValueSet set = uniqueValueSerializationStrategy.load( applicationScope,type, fields );
+ final UUID startTime = UUIDGenerator.newTimeUUID();
- //Short circut if we don't have any uniqueValues from the given fields.
- if(!set.iterator().hasNext()){
- return new MutableFieldSet( 0 );
- }
+ //Get back set of unique values that correspond to collection of fields
+ UniqueValueSet set = uniqueValueSerializationStrategy.load( applicationScope, type, fields1 );
+ //Short circuit if we don't have any uniqueValues from the given fields.
+ if ( !set.iterator().hasNext() ) {
+ return new MutableFieldSet( 0 );
+ }
- //loop through each field, and construct an entity load
- List<Id> entityIds = new ArrayList<>(fields.size());
- List<UniqueValue> uniqueValues = new ArrayList<>(fields.size());
- for(final Field expectedField: fields) {
+ //loop through each field, and construct an entity load
+ List<Id> entityIds = new ArrayList<>( fields1.size() );
+ List<UniqueValue> uniqueValues = new ArrayList<>( fields1.size() );
- UniqueValue value = set.getValue(expectedField.getName());
+ for ( final Field expectedField : fields1 ) {
- if(value ==null){
- logger.debug( "Field does not correspond to a unique value" );
- }
+ UniqueValue value = set.getValue( expectedField.getName() );
- entityIds.add(value.getEntityId());
- uniqueValues.add(value);
+ if ( value == null ) {
+ logger.debug( "Field does not correspond to a unique value" );
}
- //Load a entity for each entityId we retrieved.
- final EntitySet entitySet = entitySerializationStrategy.load(applicationScope, entityIds, startTime);
-
- //now loop through and ensure the entities are there.
- final MutationBatch deleteBatch = keyspace.prepareMutationBatch();
-
- final MutableFieldSet response = new MutableFieldSet(fields.size());
+ entityIds.add( value.getEntityId() );
+ uniqueValues.add( value );
+ }
- for(final UniqueValue expectedUnique: uniqueValues) {
- final MvccEntity entity = entitySet.getEntity(expectedUnique.getEntityId());
+ //Load a entity for each entityId we retrieved.
+ final EntitySet entitySet = entitySerializationStrategy.load( applicationScope, entityIds, startTime );
- //bad unique value, delete this, it's inconsistent
- if(entity == null || !entity.getEntity().isPresent()){
- final MutationBatch valueDelete = uniqueValueSerializationStrategy.delete(applicationScope, expectedUnique);
- deleteBatch.mergeShallow(valueDelete);
- continue;
- }
+ //now loop through and ensure the entities are there.
+ final MutationBatch deleteBatch = keyspace.prepareMutationBatch();
+ final MutableFieldSet response = new MutableFieldSet( fields1.size() );
- //else add it to our result set
- response.addEntity(expectedUnique.getField(),entity);
+ for ( final UniqueValue expectedUnique : uniqueValues ) {
+ final MvccEntity entity = entitySet.getEntity( expectedUnique.getEntityId() );
+ //bad unique value, delete this, it's inconsistent
+ if ( entity == null || !entity.getEntity().isPresent() ) {
+ final MutationBatch valueDelete =
+ uniqueValueSerializationStrategy.delete( applicationScope, expectedUnique );
+ deleteBatch.mergeShallow( valueDelete );
+ continue;
}
- //TODO: explore making this an Async process
- //We'll repair it again if we have to
- deleteBatch.execute();
- return response;
+ //else add it to our result set
+ response.addEntity( expectedUnique.getField(), entity );
+ }
+ //TODO: explore making this an Async process
+ //We'll repair it again if we have to
+ deleteBatch.execute();
- }
- catch ( ConnectionException e ) {
- logger.error( "Failed to getIdField", e );
- throw new RuntimeException( e );
- }
+ return response;
+ }
+ catch ( ConnectionException e ) {
+ logger.error( "Failed to getIdField", e );
+ throw new RuntimeException( e );
}
} );
- }
+ return ObservableTimer.time( fieldSetObservable, fieldEntityTimer );
+ }
// fire the stages
public Observable<CollectionIoEvent<MvccEntity>> stageRunner( CollectionIoEvent<Entity> writeData,
WriteStart writeState ) {
- return Observable.just( writeData ).map( writeState ).doOnNext( new Action1<CollectionIoEvent<MvccEntity>>() {
-
- @Override
- public void call( final CollectionIoEvent<MvccEntity> mvccEntityCollectionIoEvent ) {
+ return Observable.just( writeData ).map( writeState ).flatMap( mvccEntityCollectionIoEvent -> {
- Observable<CollectionIoEvent<MvccEntity>> unique =
- Observable.just( mvccEntityCollectionIoEvent ).subscribeOn( rxTaskScheduler.getAsyncIOScheduler() )
- .doOnNext( writeVerifyUnique );
+ Observable<CollectionIoEvent<MvccEntity>> uniqueObservable =
+ Observable.just( mvccEntityCollectionIoEvent ).subscribeOn( rxTaskScheduler.getAsyncIOScheduler() )
+ .doOnNext( writeVerifyUnique );
- // optimistic verification
- Observable<CollectionIoEvent<MvccEntity>> optimistic =
- Observable.just( mvccEntityCollectionIoEvent ).subscribeOn( rxTaskScheduler.getAsyncIOScheduler() )
- .doOnNext( writeOptimisticVerify );
+ // optimistic verification
+ Observable<CollectionIoEvent<MvccEntity>> optimisticObservable =
+ Observable.just( mvccEntityCollectionIoEvent ).subscribeOn( rxTaskScheduler.getAsyncIOScheduler() )
+ .doOnNext( writeOptimisticVerify );
+ final Observable<CollectionIoEvent<MvccEntity>> zip =
+ Observable.zip( uniqueObservable, optimisticObservable, ( unique, optimistic ) -> optimistic );
- //wait for both to finish
- Observable.merge( unique, optimistic ).toBlocking().last();
- }
- } );
+ return zip;
+ } );
}
@Override
public Observable<VersionSet> getLatestVersion( final Collection<Id> entityIds ) {
- final Timer.Context timer = getLatestTimer.time();
- return Observable.create( new Observable.OnSubscribe<VersionSet>() {
+
+ final Observable<VersionSet> observable = Observable.create( new Observable.OnSubscribe<VersionSet>() {
@Override
public void call( final Subscriber<? super VersionSet> subscriber ) {
try {
final VersionSet logEntries = mvccLogEntrySerializationStrategy
- .load( applicationScope, entityIds, UUIDGenerator.newTimeUUID() );
+ .load( applicationScope, entityIds, UUIDGenerator.newTimeUUID() );
subscriber.onNext( logEntries );
subscriber.onCompleted();
@@ -487,13 +421,9 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
subscriber.onError( e );
}
}
- } )
- .doOnCompleted( new Action0() {
- @Override
- public void call() {
- timer.stop();
- }
- } );
+ } );
+
+ return ObservableTimer.time( observable, getLatestTimer );
}
@@ -502,11 +432,11 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
try {
ColumnFamily<String, String> CF_SYSTEM_LOCAL =
- new ColumnFamily<String, String>( "system.local", StringSerializer.get(), StringSerializer.get(),
- StringSerializer.get() );
+ new ColumnFamily<String, String>( "system.local", StringSerializer.get(), StringSerializer.get(),
+ StringSerializer.get() );
OperationResult<CqlResult<String, String>> result =
- keyspace.prepareQuery( CF_SYSTEM_LOCAL ).withCql( "SELECT now() FROM system.local;" ).execute();
+ keyspace.prepareQuery( CF_SYSTEM_LOCAL ).withCql( "SELECT now() FROM system.local;" ).execute();
if ( result.getResult().getRows().size() == 1 ) {
return Health.GREEN;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3e2afe23/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityDeletedTask.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityDeletedTask.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityDeletedTask.java
deleted file mode 100644
index 1753d26..0000000
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityDeletedTask.java
+++ /dev/null
@@ -1,137 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.usergrid.persistence.collection.impl;
-
-
-import java.util.Set;
-import java.util.UUID;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.usergrid.persistence.collection.event.EntityDeleted;
-import org.apache.usergrid.persistence.collection.serialization.MvccEntitySerializationStrategy;
-import org.apache.usergrid.persistence.collection.serialization.MvccLogEntrySerializationStrategy;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.core.task.Task;
-import org.apache.usergrid.persistence.model.entity.Id;
-
-import com.google.inject.Inject;
-import com.google.inject.assistedinject.Assisted;
-import com.netflix.astyanax.MutationBatch;
-
-import rx.Observable;
-import rx.schedulers.Schedulers;
-
-
-/**
- * Fires Cleanup Task
- */
-public class EntityDeletedTask implements Task<Void> {
- private static final Logger LOG = LoggerFactory.getLogger(EntityDeletedTask.class);
-
- private final EntityVersionTaskFactory entityVersionTaskFactory;
- private final MvccLogEntrySerializationStrategy logEntrySerializationStrategy;
- private final MvccEntitySerializationStrategy entitySerializationStrategy;
- private final Set<EntityDeleted> listeners;
- private final ApplicationScope collectionScope;
- private final Id entityId;
- private final UUID version;
-
-
- @Inject
- public EntityDeletedTask(
- EntityVersionTaskFactory entityVersionTaskFactory,
- final MvccLogEntrySerializationStrategy logEntrySerializationStrategy,
- final MvccEntitySerializationStrategy entitySerializationStrategy,
- final Set<EntityDeleted> listeners, // MUST be a set or Guice will not inject
- @Assisted final ApplicationScope collectionScope,
- @Assisted final Id entityId,
- @Assisted final UUID version) {
-
- this.entityVersionTaskFactory = entityVersionTaskFactory;
- this.logEntrySerializationStrategy = logEntrySerializationStrategy;
- this.entitySerializationStrategy = entitySerializationStrategy;
- this.listeners = listeners;
- this.collectionScope = collectionScope;
- this.entityId = entityId;
- this.version = version;
- }
-
-
- @Override
- public void exceptionThrown(Throwable throwable) {
- LOG.error( "Unable to run update task for collection {} with entity {} and version {}",
- new Object[] { collectionScope, entityId, version }, throwable );
- }
-
-
- @Override
- public Void rejected() {
- try {
- call();
- }
- catch ( Exception e ) {
- throw new RuntimeException( "Exception thrown in call task", e );
- }
-
- return null;
- }
-
-
- @Override
- public Void call() throws Exception {
-
- entityVersionTaskFactory.getCleanupTask( collectionScope, entityId, version, true ).call();
-
- fireEvents();
- final MutationBatch entityDelete = entitySerializationStrategy.delete(collectionScope, entityId, version);
- final MutationBatch logDelete = logEntrySerializationStrategy.delete(collectionScope, entityId, version);
-
- entityDelete.execute();
- logDelete.execute();
-//
- return null;
- }
-
-
- private void fireEvents() {
- final int listenerSize = listeners.size();
-
- if ( listenerSize == 0 ) {
- return;
- }
-
- if ( listenerSize == 1 ) {
- listeners.iterator().next().deleted( collectionScope, entityId,version );
- return;
- }
-
- LOG.debug( "Started firing {} listeners", listenerSize );
-
- //if we have more than 1, run them on the rx scheduler for a max of 10 operations at a time
- Observable.from(listeners).flatMap( currentListener -> Observable.just( currentListener ).doOnNext( listener -> {
- listener.deleted( collectionScope, entityId, version );
- } ).subscribeOn( Schedulers.io() ), 10 ).toBlocking().last();
-
- LOG.debug( "Finished firing {} listeners", listenerSize );
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3e2afe23/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTask.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTask.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTask.java
deleted file mode 100644
index b5f9085..0000000
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTask.java
+++ /dev/null
@@ -1,247 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. The ASF licenses this file to You
- * under the Apache License, Version 2.0 (the "License"); you may not
- * use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License. For additional information regarding
- * copyright in this work, please see the NOTICE file in the top level
- * directory of this distribution.
- */
-package org.apache.usergrid.persistence.collection.impl;
-
-
-import java.util.Iterator;
-import java.util.List;
-import java.util.Set;
-import java.util.UUID;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.usergrid.persistence.collection.MvccLogEntry;
-import org.apache.usergrid.persistence.collection.event.EntityVersionDeleted;
-import org.apache.usergrid.persistence.collection.serialization.MvccLogEntrySerializationStrategy;
-import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
-import org.apache.usergrid.persistence.collection.serialization.UniqueValue;
-import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
-import org.apache.usergrid.persistence.collection.serialization.impl.LogEntryIterator;
-import org.apache.usergrid.persistence.core.rx.ObservableIterator;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.core.task.Task;
-import org.apache.usergrid.persistence.model.entity.Id;
-
-import com.google.inject.Inject;
-import com.google.inject.assistedinject.Assisted;
-import com.netflix.astyanax.Keyspace;
-import com.netflix.astyanax.MutationBatch;
-import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
-
-import rx.Observable;
-import rx.functions.Action1;
-import rx.functions.Func1;
-import rx.observables.BlockingObservable;
-import rx.schedulers.Schedulers;
-
-
-/**
- * Cleans up previous versions from the specified version. Note that this means the version passed in the io event is
- * retained, the range is exclusive.
- */
-public class EntityVersionCleanupTask implements Task<Void> {
-
- private static final Logger logger = LoggerFactory.getLogger( EntityVersionCleanupTask.class );
-
- private final Set<EntityVersionDeleted> listeners;
-
- private final MvccLogEntrySerializationStrategy logEntrySerializationStrategy;
- private UniqueValueSerializationStrategy uniqueValueSerializationStrategy;
- private final Keyspace keyspace;
-
- private final SerializationFig serializationFig;
-
- private final ApplicationScope scope;
- private final Id entityId;
- private final UUID version;
- private final boolean includeVersion;
-
-
- @Inject
- public EntityVersionCleanupTask(
- final SerializationFig serializationFig,
- final MvccLogEntrySerializationStrategy logEntrySerializationStrategy,
- final UniqueValueSerializationStrategy uniqueValueSerializationStrategy,
- final Keyspace keyspace,
- final Set<EntityVersionDeleted> listeners, // MUST be a set or Guice will not inject
- @Assisted final ApplicationScope scope,
- @Assisted final Id entityId,
- @Assisted final UUID version,
- @Assisted final boolean includeVersion) {
-
- this.serializationFig = serializationFig;
- this.logEntrySerializationStrategy = logEntrySerializationStrategy;
- this.uniqueValueSerializationStrategy = uniqueValueSerializationStrategy;
- this.keyspace = keyspace;
- this.listeners = listeners;
- this.scope = scope;
- this.entityId = entityId;
- this.version = version;
-
- this.includeVersion = includeVersion;
- }
-
-
- @Override
- public void exceptionThrown( final Throwable throwable ) {
- logger.error( "Unable to run update task for collection {} with entity {} and version {}",
- new Object[] { scope, entityId, version }, throwable );
- }
-
-
- @Override
- public Void rejected() {
- //Our task was rejected meaning our queue was full. We need this operation to run,
- // so we'll run it in our current thread
- try {
- call();
- }
- catch ( Exception e ) {
- throw new RuntimeException( "Exception thrown in call task", e );
- }
-
- return null;
- }
-
-
- @Override
- public Void call() throws Exception {
- //TODO Refactor this logic into a a class that can be invoked from anywhere
- //iterate all unique values
- final BlockingObservable<Long> uniqueValueCleanup =
- Observable.create( new ObservableIterator<UniqueValue>( "Unique value load" ) {
- @Override
- protected Iterator<UniqueValue> getIterator() {
- return uniqueValueSerializationStrategy.getAllUniqueFields( scope, entityId );
- }
- } )
-
- //skip current versions
- .skipWhile( new Func1<UniqueValue, Boolean>() {
- @Override
- public Boolean call( final UniqueValue uniqueValue ) {
- return !includeVersion && version.equals( uniqueValue.getEntityVersion() );
- }
- } )
- //buffer our buffer size, then roll them all up in a single batch mutation
- .buffer( serializationFig.getBufferSize() ).doOnNext( new Action1<List<UniqueValue>>() {
- @Override
- public void call( final List<UniqueValue> uniqueValues ) {
- final MutationBatch uniqueCleanupBatch = keyspace.prepareMutationBatch();
-
-
- for ( UniqueValue value : uniqueValues ) {
- uniqueCleanupBatch.mergeShallow( uniqueValueSerializationStrategy.delete( scope, value ) );
- }
-
- try {
- uniqueCleanupBatch.execute();
- }
- catch ( ConnectionException e ) {
- throw new RuntimeException( "Unable to execute batch mutation", e );
- }
- }
- } ).subscribeOn( Schedulers.io() ).countLong().toBlocking();
-
-
- //start calling the listeners for remove log entries
- BlockingObservable<Long> versionsDeletedObservable =
-
- Observable.create( new ObservableIterator<MvccLogEntry>( "Log entry iterator" ) {
- @Override
- protected Iterator<MvccLogEntry> getIterator() {
-
- return new LogEntryIterator( logEntrySerializationStrategy, scope, entityId, version,
- serializationFig.getBufferSize() );
- }
- } )
- //skip current version
- .skipWhile( new Func1<MvccLogEntry, Boolean>() {
- @Override
- public Boolean call( final MvccLogEntry mvccLogEntry ) {
- return !includeVersion && version.equals( mvccLogEntry.getVersion() );
- }
- } )
- //buffer them for efficiency
- .buffer( serializationFig.getBufferSize() ).doOnNext( new Action1<List<MvccLogEntry>>() {
- @Override
- public void call( final List<MvccLogEntry> mvccEntities ) {
-
- fireEvents( mvccEntities );
-
- final MutationBatch logCleanupBatch = keyspace.prepareMutationBatch();
-
-
- for ( MvccLogEntry entry : mvccEntities ) {
- logCleanupBatch.mergeShallow( logEntrySerializationStrategy.delete( scope, entityId, entry.getVersion() ));
- }
-
- try {
- logCleanupBatch.execute();
- }
- catch ( ConnectionException e ) {
- throw new RuntimeException( "Unable to execute batch mutation", e );
- }
- }
- } ).subscribeOn( Schedulers.io() ).countLong().toBlocking();
-
- //wait or this to complete
- final Long removedCount = uniqueValueCleanup.last();
-
- logger.debug( "Removed unique values for {} entities of entity {}", removedCount, entityId );
-
- final Long versionCleanupCount = versionsDeletedObservable.last();
-
- logger.debug( "Removed {} previous entity versions of entity {}", versionCleanupCount, entityId );
-
- return null;
- }
-
-
- private void fireEvents( final List<MvccLogEntry> versions ) {
-
- final int listenerSize = listeners.size();
-
- if ( listenerSize == 0 ) {
- return;
- }
-
- if ( listenerSize == 1 ) {
- listeners.iterator().next().versionDeleted( scope, entityId, versions );
- return;
- }
-
- logger.debug( "Started firing {} listeners", listenerSize );
-
- //if we have more than 1, run them on the rx scheduler for a max of 8 operations at a time
-
-
- //if we have more than 1, run them on the rx scheduler for a max of 10 operations at a time
- Observable.from(listeners).flatMap( currentListener -> Observable.just( currentListener ).doOnNext( listener -> {
- listener.versionDeleted( scope, entityId, versions );
- } ).subscribeOn( Schedulers.io() ), 10 ).toBlocking().last();
-
-
-
- logger.debug( "Finished firing {} listeners", listenerSize );
- }
-}
-
-
-
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3e2afe23/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTask.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTask.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTask.java
deleted file mode 100644
index fbbcdbd..0000000
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTask.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. The ASF licenses this file to You
- * under the Apache License, Version 2.0 (the "License"); you may not
- * use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License. For additional information regarding
- * copyright in this work, please see the NOTICE file in the top level
- * directory of this distribution.
- */
-package org.apache.usergrid.persistence.collection.impl;
-
-
-import java.util.Set;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.usergrid.persistence.collection.event.EntityVersionCreated;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.core.task.Task;
-import org.apache.usergrid.persistence.model.entity.Entity;
-
-import com.google.inject.Inject;
-import com.google.inject.assistedinject.Assisted;
-
-import rx.Observable;
-import rx.schedulers.Schedulers;
-
-
-/**
- * Fires events so that all EntityVersionCreated handlers area called.
- */
-public class EntityVersionCreatedTask implements Task<Void> {
- private static final Logger logger = LoggerFactory.getLogger( EntityVersionCreatedTask.class );
-
- private Set<EntityVersionCreated> listeners;
- private final ApplicationScope collectionScope;
- private final Entity entity;
-
-
- @Inject
- public EntityVersionCreatedTask( @Assisted final ApplicationScope collectionScope,
- final Set<EntityVersionCreated> listeners,
- @Assisted final Entity entity ) {
-
- this.listeners = listeners;
- this.collectionScope = collectionScope;
- this.entity = entity;
- }
-
-
- @Override
- public void exceptionThrown( final Throwable throwable ) {
- logger.error( "Unable to run update task for collection {} with entity {} and version {}",
- new Object[] { collectionScope, entity}, throwable );
- }
-
-
- @Override
- public Void rejected() {
-
- // Our task was rejected meaning our queue was full.
- // We need this operation to run, so we'll run it in our current thread
- try {
- call();
- }
- catch ( Exception e ) {
- throw new RuntimeException( "Exception thrown in call task", e );
- }
-
- return null;
- }
-
-
- @Override
- public Void call() throws Exception {
-
- fireEvents();
- return null;
- }
-
-
- private void fireEvents() {
-
- final int listenerSize = listeners.size();
-
- if ( listenerSize == 0 ) {
- return;
- }
-
- if ( listenerSize == 1 ) {
- listeners.iterator().next().versionCreated( collectionScope, entity );
- return;
- }
-
- logger.debug( "Started firing {} listeners", listenerSize );
-
-
- Observable.from( listeners )
- .flatMap( currentListener -> Observable.just( currentListener ).doOnNext( listener -> {
- listener.versionCreated( collectionScope, entity );
- } ).subscribeOn( Schedulers.io() ), 10 ).toBlocking().last();
-
-
- logger.debug( "Finished firing {} listeners", listenerSize );
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3e2afe23/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionTaskFactory.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionTaskFactory.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionTaskFactory.java
deleted file mode 100644
index 51a4607..0000000
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionTaskFactory.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. The ASF licenses this file to You
- * under the Apache License, Version 2.0 (the "License"); you may not
- * use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License. For additional information regarding
- * copyright in this work, please see the NOTICE file in the top level
- * directory of this distribution.
- */
-package org.apache.usergrid.persistence.collection.impl;
-
-
-import java.util.UUID;
-
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.model.entity.Entity;
-import org.apache.usergrid.persistence.model.entity.Id;
-
-
-public interface EntityVersionTaskFactory {
-
- /**
- * Get a task for cleaning up latent entity data. If includeVersion = true, the passed version will be cleaned up as well
- * Otherwise this is a V-1 operation
- *
- * @param scope
- * @param entityId
- * @param version
- * @param includeVersion
- * @return
- */
- EntityVersionCleanupTask getCleanupTask( final ApplicationScope scope, final Id entityId, final UUID version,
- final boolean includeVersion );
-
- /**
- * Get an entityVersionCreatedTask
- * @param scope
- * @param entity
- * @return
- */
- EntityVersionCreatedTask getCreatedTask( final ApplicationScope scope, final Entity entity );
-
- /**
- * Get an entity deleted task
- * @param collectionScope
- * @param entityId
- * @param version
- * @return
- */
- EntityDeletedTask getDeleteTask( final ApplicationScope collectionScope, final Id entityId, final UUID version );
-
-}
[6/9] incubator-usergrid git commit: Completes refactor of
compact/delete operations
Posted by to...@apache.org.
Completes refactor of compact/delete operations
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/93bacf51
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/93bacf51
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/93bacf51
Branch: refs/heads/two-dot-o-dev
Commit: 93bacf513a532ec12b4603b5b14d2bca9cabaafa
Parents: 9a89437
Author: Todd Nine <tn...@apigee.com>
Authored: Sun May 10 04:08:16 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Sun May 10 04:59:22 2015 -0600
----------------------------------------------------------------------
.../persistence/graph/GraphManager.java | 27 +-
.../graph/impl/GraphManagerImpl.java | 451 ++++++++-----------
.../graph/impl/stage/EdgeDeleteListener.java | 2 +-
.../impl/stage/EdgeDeleteListenerImpl.java | 31 +-
.../graph/impl/stage/EdgeMetaRepair.java | 6 +-
.../graph/impl/stage/EdgeMetaRepairImpl.java | 196 ++++----
.../graph/impl/stage/NodeDeleteListener.java | 2 +-
.../graph/CommittedGraphManagerIT.java | 135 ------
.../persistence/graph/GraphManagerIT.java | 67 ++-
.../graph/StorageGraphManagerIT.java | 240 ----------
10 files changed, 346 insertions(+), 811 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/93bacf51/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/GraphManager.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/GraphManager.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/GraphManager.java
index 987a36c..6100725 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/GraphManager.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/GraphManager.java
@@ -27,8 +27,7 @@ import rx.Observable;
/**
- * Represents operations that can be performed on edges within our graph. A graph should be within an
- * ApplicationScope
+ * Represents operations that can be performed on edges within our graph. A graph should be within an ApplicationScope
*
* An Edge: is defined as the following.
*
@@ -67,25 +66,37 @@ public interface GraphManager extends CPManager {
* @param edge Mark the edge as deleted in the graph
*
*
- * EdgeDelete the edge. Implementation should also delete the incoming (reversed) edge. Only deletes the specific version
+ * Implementation should also mark the incoming (reversed) edge. Only marks the specific version
*/
Observable<Edge> markEdge( Edge edge );
/**
+ * @param edge Remove the edge in the graph
+ *
*
+ * EdgeDelete the edge. Implementation should also delete the incoming (reversed) edge. Only deletes the specific version
+ * Will only delete if the edge has marked versions
+ */
+ Observable<Edge> deleteEdge( Edge edge );
+
+ /**
* Mark the node as removed from the graph.
*
* @param node The node to remove
- * @param timestamp The timestamp to apply the delete operation. Any edges connected to this node with a timestmap
- * <= the specified time will be removed from the graph
- * @return
+ * @param timestamp The timestamp to apply the mark operation.
*/
Observable<Id> markNode( Id node, long timestamp );
/**
+ * Mark the node as removed from the graph.
+ *
+ * @param node The node to remove. This will apply a timestamp to apply the delete + compact operation. Any edges connected to this node with a timestamp
+ * <= the specified time on the mark will be removed from the graph
+ */
+ Observable<Id> compactNode( final Id node );
+
+ /**
* Get all versions of this edge where versions <= max version
- * @param edge
- * @return
*/
Observable<Edge> loadEdgeVersions( SearchByEdge edge );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/93bacf51/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java
index 9a8a00f..06cb5a1 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java
@@ -29,6 +29,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
+import org.apache.usergrid.persistence.core.metrics.ObservableTimer;
import org.apache.usergrid.persistence.core.rx.ObservableIterator;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.util.ValidationUtils;
@@ -52,19 +53,14 @@ import org.apache.usergrid.persistence.model.util.UUIDGenerator;
import com.codahale.metrics.Meter;
import com.codahale.metrics.Timer;
+import com.google.common.base.Optional;
import com.google.common.base.Preconditions;
import com.google.inject.Inject;
import com.netflix.astyanax.MutationBatch;
import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
-import rx.Notification;
import rx.Observable;
-import rx.Observer;
-import rx.Subscriber;
-import rx.functions.Action0;
-import rx.functions.Action1;
import rx.functions.Func1;
-import rx.schedulers.Schedulers;
/**
@@ -86,32 +82,19 @@ public class GraphManagerImpl implements GraphManager {
private final EdgeDeleteListener edgeDeleteListener;
private final NodeDeleteListener nodeDeleteListener;
private final Timer writeEdgeTimer;
- private final Meter writeEdgeMeter;
- private final Meter deleteEdgeMeter;
- private final Timer deleteEdgeTimer;
- private final Meter deleteNodeMeter;
- private final Timer deleteNodeTimer;
- private final Meter loadEdgesFromSourceMeter;
+ private final Timer markEdgeTimer;
+ private final Timer markNodeTimer;
private final Timer loadEdgesFromSourceTimer;
- private final Meter loadEdgesToTargetMeter;
private final Timer loadEdgesToTargetTimer;
- private final Meter loadEdgesVersionsMeter;
private final Timer loadEdgesVersionsTimer;
- private final Meter loadEdgesFromSourceByTypeMeter;
private final Timer loadEdgesFromSourceByTypeTimer;
- private final Meter loadEdgesToTargetByTypeMeter;
private final Timer loadEdgesToTargetByTypeTimer;
private final Timer getEdgeTypesFromSourceTimer;
- private final Meter getEdgeTypesFromSourceMeter;
private final Timer getIdTypesFromSourceTimer;
- private final Meter getIdTypesFromSourceMeter;
- private final Meter getEdgeTypesToTargetMeter;
private final Timer getEdgeTypesToTargetTimer;
private final Timer getIdTypesToTargetTimer;
- private final Meter getIdTypesToTargetMeter;
-
- private Observer<Integer> edgeDeleteSubcriber;
- private Observer<Integer> nodeDelete;
+ private final Timer deleteNodeTimer;
+ private final Timer deleteEdgeTimer;
private final GraphFig graphFig;
@@ -141,36 +124,24 @@ public class GraphManagerImpl implements GraphManager {
this.edgeDeleteListener = edgeDeleteListener;
this.nodeDeleteListener = nodeDeleteListener;
- this.edgeDeleteSubcriber = MetricSubscriber.INSTANCE;
- this.nodeDelete = MetricSubscriber.INSTANCE;
- this.writeEdgeMeter = metricsFactory.getMeter( GraphManagerImpl.class, "write.edge.meter" );
- this.writeEdgeTimer = metricsFactory.getTimer( GraphManagerImpl.class, "write.edge.timer" );
- this.deleteEdgeMeter = metricsFactory.getMeter( GraphManagerImpl.class, "delete.edge.meter" );
- this.deleteEdgeTimer = metricsFactory.getTimer( GraphManagerImpl.class, "delete.edge.timer" );
- this.deleteNodeMeter = metricsFactory.getMeter( GraphManagerImpl.class, "delete.node.meter" );
- this.deleteNodeTimer = metricsFactory.getTimer( GraphManagerImpl.class, "delete.node.timer" );
- this.loadEdgesFromSourceMeter = metricsFactory.getMeter( GraphManagerImpl.class, "load.from.meter" );
- this.loadEdgesFromSourceTimer = metricsFactory.getTimer( GraphManagerImpl.class, "load.from.timer" );
- this.loadEdgesToTargetMeter = metricsFactory.getMeter( GraphManagerImpl.class, "load.to.meter" );
- this.loadEdgesToTargetTimer = metricsFactory.getTimer( GraphManagerImpl.class, "load.to.timer" );
- this.loadEdgesVersionsMeter = metricsFactory.getMeter( GraphManagerImpl.class, "load.versions.meter" );
- this.loadEdgesVersionsTimer = metricsFactory.getTimer( GraphManagerImpl.class, "load.versions.timer" );
- this.loadEdgesFromSourceByTypeMeter = metricsFactory.getMeter( GraphManagerImpl.class, "load.from.type.meter" );
- this.loadEdgesFromSourceByTypeTimer = metricsFactory.getTimer( GraphManagerImpl.class, "load.from.type.timer" );
- this.loadEdgesToTargetByTypeMeter = metricsFactory.getMeter( GraphManagerImpl.class, "load.to.type.meter" );
- this.loadEdgesToTargetByTypeTimer = metricsFactory.getTimer( GraphManagerImpl.class, "load.to.type.timer" );
-
- this.getEdgeTypesFromSourceTimer = metricsFactory.getTimer( GraphManagerImpl.class, "get.edge.from.timer" );
- this.getEdgeTypesFromSourceMeter = metricsFactory.getMeter( GraphManagerImpl.class, "get.edge.from.meter" );
-
- this.getIdTypesFromSourceTimer = metricsFactory.getTimer( GraphManagerImpl.class, "get.idtype.from.timer" );
- this.getIdTypesFromSourceMeter = metricsFactory.getMeter( GraphManagerImpl.class, "get.idtype.from.meter" );
-
- this.getEdgeTypesToTargetTimer = metricsFactory.getTimer( GraphManagerImpl.class, "get.edge.to.timer" );
- this.getEdgeTypesToTargetMeter = metricsFactory.getMeter( GraphManagerImpl.class, "get.edge.to.meter" );
-
- this.getIdTypesToTargetTimer = metricsFactory.getTimer( GraphManagerImpl.class, "get.idtype.to.timer" );
- this.getIdTypesToTargetMeter = metricsFactory.getMeter( GraphManagerImpl.class, "get.idtype.to.meter" );
+ this.writeEdgeTimer = metricsFactory.getTimer( GraphManagerImpl.class, "write.edge" );
+ this.markEdgeTimer = metricsFactory.getTimer( GraphManagerImpl.class, "mark.edge" );
+ this.deleteEdgeTimer = metricsFactory.getTimer( GraphManagerImpl.class, "delete.edge" );
+ this.markNodeTimer = metricsFactory.getTimer( GraphManagerImpl.class, "mark.node" );
+ this.deleteNodeTimer = metricsFactory.getTimer( GraphManagerImpl.class, "delete.node" );
+ this.loadEdgesFromSourceTimer = metricsFactory.getTimer( GraphManagerImpl.class, "load.from" );
+ this.loadEdgesToTargetTimer = metricsFactory.getTimer( GraphManagerImpl.class, "load.to" );
+ this.loadEdgesVersionsTimer = metricsFactory.getTimer( GraphManagerImpl.class, "load.versions" );
+ this.loadEdgesFromSourceByTypeTimer = metricsFactory.getTimer( GraphManagerImpl.class, "load.from.type" );
+ this.loadEdgesToTargetByTypeTimer = metricsFactory.getTimer( GraphManagerImpl.class, "load.to.type" );
+
+ this.getEdgeTypesFromSourceTimer = metricsFactory.getTimer( GraphManagerImpl.class, "get.edge.from" );
+
+ this.getIdTypesFromSourceTimer = metricsFactory.getTimer( GraphManagerImpl.class, "get.idtype.from" );
+
+ this.getEdgeTypesToTargetTimer = metricsFactory.getTimer( GraphManagerImpl.class, "get.edge.to" );
+
+ this.getIdTypesToTargetTimer = metricsFactory.getTimer( GraphManagerImpl.class, "get.idtype.to" );
}
@@ -179,42 +150,29 @@ public class GraphManagerImpl implements GraphManager {
GraphValidation.validateEdge( edge );
final MarkedEdge markedEdge = new SimpleMarkedEdge( edge, false );
- final Timer.Context timer = writeEdgeTimer.time();
- final Meter meter = writeEdgeMeter;
-
- return Observable.just( markedEdge ).map( new Func1<MarkedEdge, Edge>() {
- @Override
- public Edge call( final MarkedEdge edge ) {
- final UUID timestamp = UUIDGenerator.newTimeUUID();
+ final Observable<Edge> observable = Observable.just( markedEdge ).map( edge1 -> {
+ final UUID timestamp = UUIDGenerator.newTimeUUID();
- final MutationBatch mutation = edgeMetadataSerialization.writeEdge( scope, edge );
- final MutationBatch edgeMutation = storageEdgeSerialization.writeEdge( scope, edge, timestamp );
+ final MutationBatch mutation = edgeMetadataSerialization.writeEdge( scope, edge1 );
- mutation.mergeShallow( edgeMutation );
+ final MutationBatch edgeMutation = storageEdgeSerialization.writeEdge( scope, edge1, timestamp );
- try {
- mutation.execute();
- }
- catch ( ConnectionException e ) {
- throw new RuntimeException( "Unable to execute mutation", e );
- }
+ mutation.mergeShallow( edgeMutation );
- return edge;
- }
- } ).doOnEach( new Action1<Notification<? super Edge>>() {
- @Override
- public void call( Notification<? super Edge> notification ) {
- meter.mark();
+ try {
+ mutation.execute();
}
- } ).doOnCompleted( new Action0() {
- @Override
- public void call() {
- timer.stop();
+ catch ( ConnectionException e ) {
+ throw new RuntimeException( "Unable to execute mutation", e );
}
+
+ return edge1;
} );
+
+ return ObservableTimer.time( observable, writeEdgeTimer );
}
@@ -224,52 +182,64 @@ public class GraphManagerImpl implements GraphManager {
final MarkedEdge markedEdge = new SimpleMarkedEdge( edge, true );
- final Timer.Context timer = deleteEdgeTimer.time();
- final Meter meter = deleteEdgeMeter;
- return Observable.just( markedEdge ).map( new Func1<MarkedEdge, Edge>() {
- @Override
- public Edge call( final MarkedEdge edge ) {
+ final Observable<Edge> observable = Observable.just( markedEdge ).map( edge1 -> {
- final UUID timestamp = UUIDGenerator.newTimeUUID();
+ final UUID timestamp = UUIDGenerator.newTimeUUID();
- final MutationBatch edgeMutation = storageEdgeSerialization.writeEdge( scope, edge, timestamp );
+ final MutationBatch edgeMutation = storageEdgeSerialization.writeEdge( scope, edge1, timestamp );
- LOG.debug( "Marking edge {} as deleted to commit log", edge );
- try {
- edgeMutation.execute();
- }
- catch ( ConnectionException e ) {
- throw new RuntimeException( "Unable to execute mutation", e );
- }
+ LOG.debug( "Marking edge {} as deleted to commit log", edge1 );
+ try {
+ edgeMutation.execute();
+ }
+ catch ( ConnectionException e ) {
+ throw new RuntimeException( "Unable to execute mutation", e );
+ }
- //HystrixCassandra.async( edgeDeleteListener.receive( scope, markedEdge,
- // timestamp )).subscribeOn( Schedulers.io() ).subscribe( edgeDeleteSubcriber );
- edgeDeleteListener.receive( scope, markedEdge, timestamp ).subscribeOn( Schedulers.io() )
- .subscribe( edgeDeleteSubcriber );
+ return edge1;
+ } );
+ return ObservableTimer.time( observable, markEdgeTimer );
+ }
- return edge;
- }
- } ).doOnEach( notification -> {
- meter.mark();
- } ).doOnCompleted( () -> timer.stop() );
+
+ @Override
+ public Observable<Edge> deleteEdge( final Edge edge ) {
+
+ GraphValidation.validateEdge( edge );
+ final UUID startTimestamp = UUIDGenerator.newTimeUUID();
+
+
+ final Observable<Edge> observable =
+ Observable.create( new ObservableIterator<MarkedEdge>( "read edge versions" ) {
+ @Override
+ protected Iterator<MarkedEdge> getIterator() {
+ return storageEdgeSerialization.getEdgeVersions( scope,
+ new SimpleSearchByEdge( edge.getSourceNode(), edge.getType(), edge.getTargetNode(),
+ Long.MAX_VALUE, SearchByEdgeType.Order.ASCENDING, Optional.absent() ) );
+ }
+ } ).filter( markedEdge -> markedEdge.isDeleted() ).flatMap( marked ->
+ //fire our delete listener and wait for the results
+ edgeDeleteListener.receive( scope, marked, startTimestamp ).doOnNext(
+ //log them
+ count -> LOG.debug( "removed {} types for edge {} ", count, edge ) )
+ //return the marked edge
+ .map( count -> marked ) );
+
+
+ return ObservableTimer.time( observable, deleteEdgeTimer );
}
@Override
public Observable<Id> markNode( final Id node, final long timestamp ) {
- final Timer.Context timer = deleteNodeTimer.time();
- final Meter meter = deleteNodeMeter;
- return Observable.just( node ).map( id -> {
+ final Observable<Id> idObservable = Observable.just( node ).map( id -> {
//mark the node as deleted
-
- final UUID eventTimestamp = UUIDGenerator.newTimeUUID();
-
final MutationBatch nodeMutation = nodeSerialization.mark( scope, id, timestamp );
@@ -282,161 +252,166 @@ public class GraphManagerImpl implements GraphManager {
}
- //HystrixCassandra.async(nodeDeleteListener.receive(scope, id, eventTimestamp )).subscribeOn(
- // Schedulers.io() ).subscribe( nodeDelete );
- nodeDeleteListener.receive( scope, id, eventTimestamp ).subscribeOn( Schedulers.io() )
- .subscribe( nodeDelete );
-
return id;
- } ).doOnEach( notification -> {
- meter.mark();
- } ).doOnCompleted( () -> timer.stop() );
+ } );
+
+ return ObservableTimer.time( idObservable, markNodeTimer );
+ }
+
+
+ @Override
+ public Observable<Id> compactNode( final Id inputNode ) {
+
+
+ final UUID startTime = UUIDGenerator.newTimeUUID();
+
+
+ final Observable<Id> nodeObservable =
+ Observable.just( inputNode ).map( node -> nodeSerialization.getMaxVersion( scope, inputNode ) ).takeWhile(
+ maxTimestamp -> maxTimestamp.isPresent() )
+
+ //map our delete listener
+ .flatMap( timestamp -> nodeDeleteListener.receive( scope, inputNode, startTime ) )
+ //set to 0 if nothing is emitted
+ .lastOrDefault( 0 )
+ //log for posterity
+ .doOnNext( count -> LOG.debug( "Removed {} edges from node {}", count, inputNode ) )
+ //return our id
+ .map( count -> inputNode );
+
+ return ObservableTimer.time( nodeObservable, this.deleteNodeTimer );
}
@Override
public Observable<Edge> loadEdgeVersions( final SearchByEdge searchByEdge ) {
- final Timer.Context timer = loadEdgesVersionsTimer.time();
- final Meter meter = loadEdgesVersionsMeter;
- return Observable.create( new ObservableIterator<MarkedEdge>( "getEdgeTypesFromSource" ) {
- @Override
- protected Iterator<MarkedEdge> getIterator() {
- return storageEdgeSerialization.getEdgeVersions( scope, searchByEdge );
- }
- } ).buffer( graphFig.getScanPageSize() )
- .flatMap( new EdgeBufferFilter( searchByEdge.getMaxTimestamp(), searchByEdge.filterMarked() ) )
- .cast( Edge.class ).doOnEach( notification -> {
- meter.mark();
- } ).doOnCompleted( () -> timer.stop() );
+
+ final Observable<Edge> edges =
+ Observable.create( new ObservableIterator<MarkedEdge>( "getEdgeTypesFromSource" ) {
+ @Override
+ protected Iterator<MarkedEdge> getIterator() {
+ return storageEdgeSerialization.getEdgeVersions( scope, searchByEdge );
+ }
+ } ).buffer( graphFig.getScanPageSize() )
+ .flatMap( new EdgeBufferFilter( searchByEdge.getMaxTimestamp(), searchByEdge.filterMarked() ) );
+
+ return ObservableTimer.time( edges, loadEdgesVersionsTimer );
}
@Override
public Observable<Edge> loadEdgesFromSource( final SearchByEdgeType search ) {
- final Timer.Context timer = loadEdgesFromSourceTimer.time();
- final Meter meter = loadEdgesFromSourceMeter;
- return Observable.create( new ObservableIterator<MarkedEdge>( "getEdgeTypesFromSource" ) {
- @Override
- protected Iterator<MarkedEdge> getIterator() {
- return storageEdgeSerialization.getEdgesFromSource( scope, search );
- }
- } ).buffer( graphFig.getScanPageSize() )
- .flatMap( new EdgeBufferFilter( search.getMaxTimestamp(), search.filterMarked() ) ).cast( Edge.class )
- .doOnEach( notification -> {
- meter.mark();
- } ).doOnCompleted( () -> timer.stop() );
+ final Observable<Edge> edges =
+ Observable.create( new ObservableIterator<MarkedEdge>( "getEdgeTypesFromSource" ) {
+ @Override
+ protected Iterator<MarkedEdge> getIterator() {
+ return storageEdgeSerialization.getEdgesFromSource( scope, search );
+ }
+ } ).buffer( graphFig.getScanPageSize() )
+ .flatMap( new EdgeBufferFilter( search.getMaxTimestamp(), search.filterMarked() ) );
+
+ return ObservableTimer.time( edges, loadEdgesFromSourceTimer );
}
@Override
public Observable<Edge> loadEdgesToTarget( final SearchByEdgeType search ) {
- final Timer.Context timer = loadEdgesToTargetTimer.time();
- final Meter meter = loadEdgesToTargetMeter;
- return Observable.create( new ObservableIterator<MarkedEdge>( "getEdgeTypesFromSource" ) {
- @Override
- protected Iterator<MarkedEdge> getIterator() {
- return storageEdgeSerialization.getEdgesToTarget( scope, search );
- }
- } ).buffer( graphFig.getScanPageSize() )
- .flatMap( new EdgeBufferFilter( search.getMaxTimestamp(), search.filterMarked() ) ).cast( Edge.class )
- .doOnEach( notification -> {
- meter.mark();
- } ).doOnCompleted( () -> timer.stop() );
+ final Observable<Edge> edges =
+ Observable.create( new ObservableIterator<MarkedEdge>( "getEdgeTypesFromSource" ) {
+ @Override
+ protected Iterator<MarkedEdge> getIterator() {
+ return storageEdgeSerialization.getEdgesToTarget( scope, search );
+ }
+ } ).buffer( graphFig.getScanPageSize() )
+ .flatMap( new EdgeBufferFilter( search.getMaxTimestamp(), search.filterMarked() ) );
+
+
+ return ObservableTimer.time( edges, loadEdgesToTargetTimer );
}
@Override
public Observable<Edge> loadEdgesFromSourceByType( final SearchByIdType search ) {
- final Timer.Context timer = loadEdgesFromSourceByTypeTimer.time();
- final Meter meter = loadEdgesFromSourceByTypeMeter;
- return Observable.create( new ObservableIterator<MarkedEdge>( "getEdgeTypesFromSource" ) {
- @Override
- protected Iterator<MarkedEdge> getIterator() {
- return storageEdgeSerialization.getEdgesFromSourceByTargetType( scope, search );
- }
- } ).buffer( graphFig.getScanPageSize() )
- .flatMap( new EdgeBufferFilter( search.getMaxTimestamp(), search.filterMarked() ) )
-
- .cast( Edge.class ).doOnEach( notification -> {
- meter.mark();
- } ).doOnCompleted( () -> timer.stop() );
+ final Observable<Edge> edges =
+ Observable.create( new ObservableIterator<MarkedEdge>( "getEdgeTypesFromSource" ) {
+ @Override
+ protected Iterator<MarkedEdge> getIterator() {
+ return storageEdgeSerialization.getEdgesFromSourceByTargetType( scope, search );
+ }
+ } ).buffer( graphFig.getScanPageSize() )
+ .flatMap( new EdgeBufferFilter( search.getMaxTimestamp(), search.filterMarked() ) );
+
+ return ObservableTimer.time( edges, loadEdgesFromSourceTimer );
}
@Override
public Observable<Edge> loadEdgesToTargetByType( final SearchByIdType search ) {
- final Timer.Context timer = loadEdgesToTargetByTypeTimer.time();
- final Meter meter = loadEdgesToTargetByTypeMeter;
- return Observable.create( new ObservableIterator<MarkedEdge>( "getEdgeTypesFromSource" ) {
- @Override
- protected Iterator<MarkedEdge> getIterator() {
- return storageEdgeSerialization.getEdgesToTargetBySourceType( scope, search );
- }
- } ).buffer( graphFig.getScanPageSize() )
- .flatMap( new EdgeBufferFilter( search.getMaxTimestamp(), search.filterMarked() ) ).cast( Edge.class )
- .doOnEach( notification -> {
- meter.mark();
- } ).doOnCompleted( () -> timer.stop() );
+ final Observable<Edge> edges =
+ Observable.create( new ObservableIterator<MarkedEdge>( "getEdgeTypesFromSource" ) {
+ @Override
+ protected Iterator<MarkedEdge> getIterator() {
+ return storageEdgeSerialization.getEdgesToTargetBySourceType( scope, search );
+ }
+ } ).buffer( graphFig.getScanPageSize() )
+ .flatMap( new EdgeBufferFilter( search.getMaxTimestamp(), search.filterMarked() ) );
+
+ return ObservableTimer.time( edges, loadEdgesToTargetByTypeTimer );
}
@Override
public Observable<String> getEdgeTypesFromSource( final SearchEdgeType search ) {
- final Timer.Context timer = getEdgeTypesFromSourceTimer.time();
- final Meter meter = getEdgeTypesFromSourceMeter;
- return Observable.create( new ObservableIterator<String>( "getEdgeTypesFromSource" ) {
- @Override
- protected Iterator<String> getIterator() {
- return edgeMetadataSerialization.getEdgeTypesFromSource( scope, search );
- }
- } ).doOnEach( notification -> {
- meter.mark();
- } ).doOnCompleted( () -> timer.stop() );
+ final Observable<String> edgeTypes = Observable.create(
+ new ObservableIterator<String>( "getEdgeTypesFromSource" ) {
+ @Override
+ protected Iterator<String> getIterator() {
+ return edgeMetadataSerialization.getEdgeTypesFromSource( scope, search );
+ }
+ } );
+
+ return ObservableTimer.time( edgeTypes, getEdgeTypesFromSourceTimer );
}
@Override
public Observable<String> getIdTypesFromSource( final SearchIdType search ) {
- final Timer.Context timer = getIdTypesFromSourceTimer.time();
- final Meter meter = getIdTypesFromSourceMeter;
- return Observable.create( new ObservableIterator<String>( "getIdTypesFromSource" ) {
+ final Observable<String> edgeTypes = Observable.create( new ObservableIterator<String>( "getIdTypesFromSource" ) {
@Override
protected Iterator<String> getIterator() {
return edgeMetadataSerialization.getIdTypesFromSource( scope, search );
}
- } ).doOnEach( notification -> {
- meter.mark();
- } ).doOnCompleted( () -> timer.stop() );
+ } );
+
+ return ObservableTimer.time( edgeTypes, getIdTypesFromSourceTimer );
}
@Override
public Observable<String> getEdgeTypesToTarget( final SearchEdgeType search ) {
- final Timer.Context timer = getEdgeTypesToTargetTimer.time();
- final Meter meter = getEdgeTypesToTargetMeter;
- return Observable.create( new ObservableIterator<String>( "getEdgeTypesToTarget" ) {
- @Override
- protected Iterator<String> getIterator() {
- return edgeMetadataSerialization.getEdgeTypesToTarget( scope, search );
- }
- } ).doOnEach( notification -> {
- meter.mark();
- } ).doOnCompleted( () -> timer.stop() );
+ final Observable<String> edgeTypes = Observable.create(
+ new ObservableIterator<String>( "getEdgeTypesToTarget" ) {
+ @Override
+ protected Iterator<String> getIterator() {
+ return edgeMetadataSerialization.getEdgeTypesToTarget( scope, search );
+ }
+ } );
+
+ return ObservableTimer.time( edgeTypes, getEdgeTypesFromSourceTimer );
}
@Override
public Observable<String> getIdTypesToTarget( final SearchIdType search ) {
- final Timer.Context timer = getIdTypesToTargetTimer.time();
- final Meter meter = getIdTypesToTargetMeter;
- return Observable.create( new ObservableIterator<String>( "getIdTypesToTarget" ) {
- @Override
- protected Iterator<String> getIterator() {
- return edgeMetadataSerialization.getIdTypesToTarget( scope, search );
- }
- } ).doOnEach( notification -> {
- meter.mark();
- } ).doOnCompleted( () -> timer.stop() );
+ final Observable<String> edgeTypes = Observable.create(
+ new ObservableIterator<String>( "getIdTypesToTarget" ) {
+ @Override
+ protected Iterator<String> getIterator() {
+ return edgeMetadataSerialization.getIdTypesToTarget( scope, search );
+ }
+ } );
+
+ return ObservableTimer.time( edgeTypes, getIdTypesToTargetTimer );
}
@@ -472,7 +447,7 @@ public class GraphManagerImpl implements GraphManager {
final long edgeTimestamp = edge.getTimestamp();
//our edge needs to not be deleted and have a version that's > max Version
- if ( edge.isDeleted() || Long.compare( edgeTimestamp, maxTimestamp ) > 0 ) {
+ if (( edge.isDeleted() && filterMarked) || Long.compare( edgeTimestamp, maxTimestamp ) > 0 ) {
return false;
}
@@ -498,52 +473,4 @@ public class GraphManagerImpl implements GraphManager {
} );
}
}
-
-
- /**
- * Set the subscription for the edge delete
- */
- public void setEdgeDeleteSubcriber( final Observer<Integer> edgeDeleteSubcriber ) {
- Preconditions.checkNotNull( edgeDeleteSubcriber, "Subscriber cannot be null" );
- this.edgeDeleteSubcriber = edgeDeleteSubcriber;
- }
-
-
- /**
- * Set the subscription for the node delete
- */
- public void setNodeDelete( final Observer<Integer> nodeDelete ) {
- Preconditions.checkNotNull( nodeDelete, "Subscriber cannot be null" );
- this.nodeDelete = nodeDelete;
- }
-
-
- /**
- * Simple subscriber that can be used to gather metrics. Needs to be refactored to use codehale metrics
- */
- private static class MetricSubscriber extends Subscriber<Integer> {
-
-
- private static final MetricSubscriber INSTANCE = new MetricSubscriber();
-
- private final Logger logger = LoggerFactory.getLogger( MetricSubscriber.class );
-
-
- @Override
- public void onCompleted() {
- logger.debug( "Event completed" );
- }
-
-
- @Override
- public void onError( final Throwable e ) {
- logger.error( "Failed to execute event", e );
- }
-
-
- @Override
- public void onNext( final Integer integer ) {
- logger.debug( "Next received {}", integer );
- }
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/93bacf51/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeDeleteListener.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeDeleteListener.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeDeleteListener.java
index 4b53940..d8a8896 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeDeleteListener.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeDeleteListener.java
@@ -40,7 +40,7 @@ public interface EdgeDeleteListener {
* @param eventTimestamp
* @return
*/
- public Observable<Integer> receive( final ApplicationScope scope, final MarkedEdge edge, final UUID eventTimestamp );
+ Observable<Integer> receive( final ApplicationScope scope, final MarkedEdge edge, final UUID eventTimestamp );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/93bacf51/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeDeleteListenerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeDeleteListenerImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeDeleteListenerImpl.java
index 7e42293..22a5ad0 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeDeleteListenerImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeDeleteListenerImpl.java
@@ -61,27 +61,16 @@ public class EdgeDeleteListenerImpl implements EdgeDeleteListener {
return edgeDeleteRepair.repair( scope, edge, eventTimestamp )
- .flatMap( new Func1<MarkedEdge, Observable<Integer>>() {
- @Override
- public Observable<Integer> call( final MarkedEdge markedEdge ) {
-
- Observable<Integer> sourceDelete = edgeMetaRepair
- .repairSources( scope, edge.getSourceNode(), edge.getType(),
- maxTimestamp );
-
- Observable<Integer> targetDelete = edgeMetaRepair
- .repairTargets( scope, edge.getTargetNode(), edge.getType(),
- maxTimestamp );
-
- return Observable.zip( sourceDelete, targetDelete,
- new Func2<Integer, Integer, Integer>() {
- @Override
- public Integer call( final Integer sourceCount,
- final Integer targetCount ) {
- return sourceCount + targetCount;
- }
- } );
- }
+ .flatMap( markedEdge -> {
+
+ Observable<Integer> sourceDelete = edgeMetaRepair
+ .repairSources( scope, edge.getSourceNode(), edge.getType(), maxTimestamp );
+
+ Observable<Integer> targetDelete = edgeMetaRepair
+ .repairTargets( scope, edge.getTargetNode(), edge.getType(), maxTimestamp );
+
+ return Observable.zip( sourceDelete, targetDelete,
+ ( sourceCount, targetCount ) -> sourceCount + targetCount );
} );
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/93bacf51/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeMetaRepair.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeMetaRepair.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeMetaRepair.java
index 5ba822b..efd88fc 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeMetaRepair.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeMetaRepair.java
@@ -27,7 +27,7 @@ import rx.Observable;
/**
- * Audits edge meta data and removes them if they're obscelete
+ * Audits edge meta data and removes them if they're obsolete
*/
public interface EdgeMetaRepair {
@@ -42,7 +42,7 @@ public interface EdgeMetaRepair {
* @return An observable that emits the total number of sub types still in use. 0 implies the type and subtypes
* have been removed. Anything > 0 implies the edgeType and subTypes are still in use
*/
- public Observable<Integer> repairSources( ApplicationScope scope, Id sourceId, String edgeType, long maxTimestamp );
+ Observable<Integer> repairSources( ApplicationScope scope, Id sourceId, String edgeType, long maxTimestamp );
/**
@@ -56,5 +56,5 @@ public interface EdgeMetaRepair {
* @return An observable that emits the total number of sub types still in use. 0 implies the type and subtypes
* have been removed. Anything > 0 implies the edgeType and subTypes are still in use
*/
- public Observable<Integer> repairTargets( ApplicationScope scope, Id targetId, String edgeType, long maxTimestamp );
+ Observable<Integer> repairTargets( ApplicationScope scope, Id targetId, String edgeType, long maxTimestamp );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/93bacf51/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeMetaRepairImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeMetaRepairImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeMetaRepairImpl.java
index 5e76f75..206145a 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeMetaRepairImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeMetaRepairImpl.java
@@ -118,91 +118,79 @@ public class EdgeMetaRepairImpl implements EdgeMetaRepair {
Observable<Integer> deleteCounts = serialization.loadEdgeSubTypes( scope, node, edgeType, maxTimestamp ).buffer(
graphFig.getRepairConcurrentSize() )
//buffer them into concurrent groups based on the concurrent repair size
- .flatMap( new Func1<List<String>, Observable<Integer>>() {
-
- @Override
- public Observable<Integer> call( final List<String> types ) {
-
-
- final MutationBatch batch = keyspace.prepareMutationBatch();
-
- final List<Observable<Integer>> checks = new ArrayList<Observable<Integer>>( types.size() );
-
- //for each id type, check if the exist in parallel to increase processing speed
- for ( final String subType : types ) {
-
- LOG.debug( "Checking for edges with nodeId {}, type {}, and subtype {}", node, edgeType,
- subType );
-
- Observable<Integer> search =
- //load each edge in it's own thread
- serialization.loadEdges( scope, node, edgeType, subType, maxTimestamp )
- .doOnNext( RX_LOG ).take( 1 ).count()
- .doOnNext( new Action1<Integer>() {
-
- @Override
- public void call( final Integer count ) {
- /**
- * we only want to delete if no edges are in this class. If
- * there
- * are
- * still edges
- * we must retain the information in order to keep our index
- * structure
- * correct for edge
- * iteration
- **/
- if ( count != 0 ) {
- LOG.debug( "Found edge with nodeId {}, type {}, "
- + "and subtype {}. Not removing subtype. ",
- node, edgeType, subType );
- return;
- }
-
-
- LOG.debug( "No edges with nodeId {}, type {}, "
- + "and subtype {}. Removing subtype.", node,
- edgeType, subType );
- batch.mergeShallow( serialization
- .removeEdgeSubType( scope, node, edgeType, subType,
- maxTimestamp ) );
- }
- } );
-
- checks.add( search );
- }
-
-
- /**
- * Sum up the total number of edges we had, then execute the mutation if we have
- * anything to do
- */
-
-
- return MathObservable.sumInteger( Observable.merge( checks ) )
- .doOnNext( new Action1<Integer>() {
- @Override
- public void call( final Integer count ) {
-
-
- LOG.debug(
- "Executing batch for subtype deletion with " +
- "type {}. "
- + "Mutation has {} rows to mutate ",
- edgeType, batch.getRowCount() );
-
- try {
- batch.execute();
- }
- catch ( ConnectionException e ) {
- throw new RuntimeException( "Unable to connect to casandra", e );
- }
- }
- }
-
-
- );
+ .flatMap( types -> {
+
+
+ final MutationBatch batch = keyspace.prepareMutationBatch();
+
+ final List<Observable<Integer>> checks = new ArrayList<Observable<Integer>>( types.size() );
+
+ //for each id type, check if the exist in parallel to increase processing speed
+ for ( final String subType : types ) {
+
+ LOG.debug( "Checking for edges with nodeId {}, type {}, and subtype {}", node, edgeType,
+ subType );
+
+ Observable<Integer> search =
+ //load each edge in it's own thread
+ serialization.loadEdges( scope, node, edgeType, subType, maxTimestamp )
+ .doOnNext( RX_LOG ).take( 1 ).count()
+ .doOnNext( count -> {
+ /**
+ * we only want to delete if no edges are in this class. If
+ * there
+ * are
+ * still edges
+ * we must retain the information in order to keep our index
+ * structure
+ * correct for edge
+ * iteration
+ **/
+ if ( count != 0 ) {
+ LOG.debug( "Found edge with nodeId {}, type {}, "
+ + "and subtype {}. Not removing subtype. ",
+ node, edgeType, subType );
+ return;
+ }
+
+
+ LOG.debug( "No edges with nodeId {}, type {}, "
+ + "and subtype {}. Removing subtype.", node,
+ edgeType, subType );
+ batch.mergeShallow( serialization
+ .removeEdgeSubType( scope, node, edgeType, subType,
+ maxTimestamp ) );
+ } );
+
+ checks.add( search );
}
+
+
+ /**
+ * Sum up the total number of edges we had, then execute the mutation if we have
+ * anything to do
+ */
+
+
+ return MathObservable.sumInteger( Observable.merge( checks ) )
+ .doOnNext( count -> {
+
+
+ LOG.debug( "Executing batch for subtype deletion with " +
+ "type {}. " + "Mutation has {} rows to mutate ",
+ edgeType, batch.getRowCount() );
+
+ try {
+ batch.execute();
+ }
+ catch ( ConnectionException e ) {
+ throw new RuntimeException(
+ "Unable to connect to casandra", e );
+ }
+ }
+
+
+ );
} )
//if we get no edges, emit a 0 so the caller knows we can delete the type
.defaultIfEmpty( 0 );
@@ -210,29 +198,25 @@ public class EdgeMetaRepairImpl implements EdgeMetaRepair {
//sum up everything emitted by sub types. If there's no edges existing for all sub types,
// then we can safely remove them
- return MathObservable.sumInteger( deleteCounts ).lastOrDefault( 0 ).doOnNext( new Action1<Integer>() {
-
- @Override
- public void call( final Integer subTypeUsedCount ) {
-
- /**
- * We can only execute deleting this type if no sub types were deleted
- */
- if ( subTypeUsedCount != 0 ) {
- LOG.debug( "Type {} has {} subtypes in use as of maxTimestamp {}. Not deleting type.", edgeType,
- subTypeUsedCount, maxTimestamp );
- return;
- }
+ return MathObservable.sumInteger( deleteCounts ).lastOrDefault( 0 ).doOnNext( subTypeUsedCount -> {
+
+ /**
+ * We can only execute deleting this type if no sub types were deleted
+ */
+ if ( subTypeUsedCount != 0 ) {
+ LOG.debug( "Type {} has {} subtypes in use as of maxTimestamp {}. Not deleting type.", edgeType,
+ subTypeUsedCount, maxTimestamp );
+ return;
+ }
- LOG.debug( "Type {} has no subtypes in use as of maxTimestamp {}. Deleting type.", edgeType,
- maxTimestamp );
- try {
- serialization.removeEdgeType( scope, node, edgeType, maxTimestamp ).execute();
- }
- catch ( ConnectionException e ) {
- throw new RuntimeException( "Unable to connect to casandra", e );
- }
+ LOG.debug( "Type {} has no subtypes in use as of maxTimestamp {}. Deleting type.", edgeType,
+ maxTimestamp );
+ try {
+ serialization.removeEdgeType( scope, node, edgeType, maxTimestamp ).execute();
+ }
+ catch ( ConnectionException e ) {
+ throw new RuntimeException( "Unable to connect to casandra", e );
}
} );
}
@@ -241,7 +225,7 @@ public class EdgeMetaRepairImpl implements EdgeMetaRepair {
/**
* Simple edge serialization
*/
- private static interface CleanSerialization {
+ private interface CleanSerialization {
/**
* Load all subtypes for the edge with a maxTimestamp <= the provided maxTimestamp
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/93bacf51/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/NodeDeleteListener.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/NodeDeleteListener.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/NodeDeleteListener.java
index 89f280d..68569e5 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/NodeDeleteListener.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/NodeDeleteListener.java
@@ -42,5 +42,5 @@ public interface NodeDeleteListener {
* @return An observable that emits the total number of edges that have been removed with this node both as the
* target and source
*/
- public Observable<Integer> receive( final ApplicationScope scope, final Id node, final UUID timestamp );
+ Observable<Integer> receive( final ApplicationScope scope, final Id node, final UUID timestamp );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/93bacf51/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/CommittedGraphManagerIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/CommittedGraphManagerIT.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/CommittedGraphManagerIT.java
deleted file mode 100644
index 658f4bf..0000000
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/CommittedGraphManagerIT.java
+++ /dev/null
@@ -1,135 +0,0 @@
-/*
- *
- * * Licensed to the Apache Software Foundation (ASF) under one
- * * or more contributor license agreements. See the NOTICE file
- * * distributed with this work for additional information
- * * regarding copyright ownership. The ASF licenses this file
- * * to you under the Apache License, Version 2.0 (the
- * * "License"); you may not use this file except in compliance
- * * with the License. You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing,
- * * software distributed under the License is distributed on an
- * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * * KIND, either express or implied. See the License for the
- * * specific language governing permissions and limitations
- * * under the License.
- *
- */
-package org.apache.usergrid.persistence.graph;
-
-
-import org.junit.runner.RunWith;
-
-import org.apache.usergrid.persistence.core.test.ITRunner;
-import org.apache.usergrid.persistence.core.test.UseModules;
-import org.apache.usergrid.persistence.graph.guice.TestGraphModule;
-import org.apache.usergrid.persistence.model.entity.Id;
-
-import rx.Observable;
-
-
-/**
- * Integration test that performs all calls immediately after writes without blocking. Tests that our
- * view is immediately consistent to our users, even if we have yet to perform background processing
- */
-@RunWith(ITRunner.class)
-@UseModules({ TestGraphModule.class })
-public class CommittedGraphManagerIT extends GraphManagerIT {
-
-
- @Override
- protected GraphManager getHelper(GraphManager gm) {
- return new ComittedGraphTestHelper( gm );
- }
-
-
- /**
- * Doesn't wait for the async process to happen before returning. Simply executes and immediately returns.
- */
- public static class ComittedGraphTestHelper implements GraphManager {
-
- private final GraphManager graphManager;
-
-
- public ComittedGraphTestHelper( final GraphManager graphManager ) {
- this.graphManager = graphManager;
- }
-
-
- @Override
- public Observable<Edge> writeEdge( final Edge edge ) {
- return graphManager.writeEdge( edge );
- }
-
-
- @Override
- public Observable<Edge> markEdge( final Edge edge ) {
- return graphManager.markEdge( edge );
- }
-
-
- @Override
- public Observable<Id> markNode( final Id node, final long timestamp ) {
- return graphManager.markNode( node, timestamp );
- }
-
-
- @Override
- public Observable<Edge> loadEdgeVersions( final SearchByEdge edge ) {
- return graphManager.loadEdgeVersions( edge );
- }
-
-
- @Override
- public Observable<Edge> loadEdgesFromSource( final SearchByEdgeType search ) {
- return graphManager.loadEdgesFromSource( search );
- }
-
-
- @Override
- public Observable<Edge> loadEdgesToTarget( final SearchByEdgeType search ) {
- return graphManager.loadEdgesToTarget( search );
- }
-
-
- @Override
- public Observable<Edge> loadEdgesFromSourceByType( final SearchByIdType search ) {
- return graphManager.loadEdgesFromSourceByType(search);
- }
-
-
- @Override
- public Observable<Edge> loadEdgesToTargetByType( final SearchByIdType search ) {
- return graphManager.loadEdgesToTargetByType( search );
- }
-
-
- @Override
- public Observable<String> getEdgeTypesFromSource( final SearchEdgeType search ) {
- return graphManager.getEdgeTypesFromSource( search );
- }
-
-
- @Override
- public Observable<String> getIdTypesFromSource( final SearchIdType search ) {
- return graphManager.getIdTypesFromSource( search );
- }
-
-
- @Override
- public Observable<String> getEdgeTypesToTarget( final SearchEdgeType search ) {
- return graphManager.getEdgeTypesToTarget( search );
- }
-
-
- @Override
- public Observable<String> getIdTypesToTarget( final SearchIdType search ) {
- return graphManager.getIdTypesToTarget( search );
- }
-
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/93bacf51/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerIT.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerIT.java
index 340c712..75f61f0 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerIT.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerIT.java
@@ -25,11 +25,15 @@ import java.util.concurrent.TimeoutException;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
+import org.junit.runner.RunWith;
import org.apache.usergrid.persistence.core.guice.MigrationManagerRule;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
+import org.apache.usergrid.persistence.core.test.ITRunner;
+import org.apache.usergrid.persistence.core.test.UseModules;
import org.apache.usergrid.persistence.core.util.IdGenerator;
+import org.apache.usergrid.persistence.graph.guice.TestGraphModule;
import org.apache.usergrid.persistence.graph.impl.SimpleSearchEdgeType;
import org.apache.usergrid.persistence.graph.impl.SimpleSearchIdType;
import org.apache.usergrid.persistence.model.entity.Id;
@@ -49,8 +53,9 @@ import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
-
-public abstract class GraphManagerIT {
+@RunWith(ITRunner.class)
+@UseModules({ TestGraphModule.class })
+public class GraphManagerIT {
@Inject
@@ -64,12 +69,6 @@ public abstract class GraphManagerIT {
protected ApplicationScope scope;
- /**
- * Get the helper for performing our tests
- *
- * @return The helper to use when writing/synchronizing tests
- */
- protected abstract GraphManager getHelper( GraphManager gm );
@Before
@@ -81,7 +80,7 @@ public abstract class GraphManagerIT {
@Test
public void testWriteReadEdgeTypeSource() throws TimeoutException, InterruptedException {
- GraphManager gm = getHelper( emf.createEdgeManager( scope ) );
+ GraphManager gm = emf.createEdgeManager( scope );
Edge edge = createEdge( "source", "test", "target" );
@@ -113,7 +112,7 @@ public abstract class GraphManagerIT {
@Test
public void testWriteReadEdgeTypeTarget() throws TimeoutException, InterruptedException {
- GraphManager gm = getHelper( emf.createEdgeManager( scope ) );
+ GraphManager gm = emf.createEdgeManager( scope );
Edge edge = createEdge( "source", "test", "target" );
@@ -145,7 +144,7 @@ public abstract class GraphManagerIT {
@Test
public void testWriteReadEdgeTypeVersionSource() throws TimeoutException, InterruptedException {
- GraphManager gm = getHelper( emf.createEdgeManager( scope ) );
+ GraphManager gm = emf.createEdgeManager( scope );
final long earlyVersion = 1000l;
@@ -179,7 +178,7 @@ public abstract class GraphManagerIT {
@Test
public void testWriteReadEdgeTypeVersionTarget() throws TimeoutException, InterruptedException {
- GraphManager gm = getHelper( emf.createEdgeManager( scope ) );
+ GraphManager gm = emf.createEdgeManager( scope );
final long earlyVersion = 10000l;
@@ -217,7 +216,7 @@ public abstract class GraphManagerIT {
@Test
public void testWriteReadEdgeTypeVersionSourceDistinct() throws TimeoutException, InterruptedException {
- GraphManager gm = getHelper( emf.createEdgeManager( scope ) );
+ GraphManager gm = emf.createEdgeManager( scope );
final long earlyVersion = 10000l;
@@ -289,7 +288,7 @@ public abstract class GraphManagerIT {
public void testWriteReadEdgeTypeVersionTargetDistinct() throws TimeoutException, InterruptedException {
- GraphManager gm = getHelper( emf.createEdgeManager( scope ) );
+ GraphManager gm = emf.createEdgeManager( scope );
final long earlyVersion = 10000l;
@@ -361,7 +360,7 @@ public abstract class GraphManagerIT {
@Test
public void testWriteReadEdgeTypePagingSource() throws TimeoutException, InterruptedException {
- GraphManager gm = getHelper( emf.createEdgeManager( scope ) );
+ GraphManager gm = emf.createEdgeManager( scope );
final Id sourceId = IdGenerator.createId( "source" );
@@ -416,7 +415,7 @@ public abstract class GraphManagerIT {
public void testWriteReadEdgeTypePagingTarget() {
- GraphManager gm = getHelper( emf.createEdgeManager( scope ) );
+ GraphManager gm = emf.createEdgeManager( scope );
final Id targetId = IdGenerator.createId( "target" );
@@ -471,7 +470,7 @@ public abstract class GraphManagerIT {
@Test
public void testWriteReadEdgeTypeTargetTypeSource() {
- GraphManager gm = getHelper( emf.createEdgeManager( scope ) );
+ GraphManager gm = emf.createEdgeManager( scope );
Edge edge = createEdge( "source", "test", "target" );
@@ -507,7 +506,7 @@ public abstract class GraphManagerIT {
@Test
public void testWriteReadEdgeTypeTargetTypeTarget() {
- GraphManager gm = getHelper( emf.createEdgeManager( scope ) );
+ GraphManager gm = emf.createEdgeManager( scope );
;
@@ -544,7 +543,7 @@ public abstract class GraphManagerIT {
@Test
public void testWriteReadEdgeDeleteSource() {
- GraphManager gm = getHelper( emf.createEdgeManager( scope ) );
+ GraphManager gm = emf.createEdgeManager( scope );
Edge edge = createEdge( "source", "test", "target" );
@@ -614,7 +613,7 @@ public abstract class GraphManagerIT {
@Test
public void testWriteReadEdgeDeleteTarget() {
- GraphManager gm = getHelper( emf.createEdgeManager( scope ) );
+ GraphManager gm = emf.createEdgeManager( scope );
Edge edge = createEdge( "source", "test", "target" );
@@ -670,7 +669,7 @@ public abstract class GraphManagerIT {
@Test
public void testWriteReadEdgeTypesSourceTypes() {
- final GraphManager gm = getHelper( emf.createEdgeManager( scope ) );
+ final GraphManager gm = emf.createEdgeManager( scope );
Id sourceId = new SimpleId( "source" );
Id targetId1 = new SimpleId( "target" );
@@ -733,7 +732,7 @@ public abstract class GraphManagerIT {
@Test
public void testWriteReadEdgeTypesTargetTypes() {
- final GraphManager gm = getHelper( emf.createEdgeManager( scope ) );
+ final GraphManager gm = emf.createEdgeManager( scope );
Id sourceId1 = new SimpleId( "source" );
Id sourceId2 = new SimpleId( "source2" );
@@ -798,7 +797,7 @@ public abstract class GraphManagerIT {
@Test
public void testWriteReadEdgeTypesSourceTypesPaging() {
- final GraphManager gm = getHelper( emf.createEdgeManager( scope ) );
+ final GraphManager gm = emf.createEdgeManager( scope );
Id sourceId1 = new SimpleId( "source" );
Id targetId1 = new SimpleId( "target" );
@@ -876,7 +875,7 @@ public abstract class GraphManagerIT {
@Test
public void testWriteReadEdgeTypesTargetTypesPaging() {
- final GraphManager gm = getHelper( emf.createEdgeManager( scope ) );
+ final GraphManager gm = emf.createEdgeManager( scope );
Id sourceId1 = new SimpleId( "source" );
Id sourceId2 = new SimpleId( "source2" );
@@ -957,7 +956,7 @@ public abstract class GraphManagerIT {
@Test
public void testMarkSourceEdges() throws InterruptedException {
- final GraphManager gm = getHelper( emf.createEdgeManager( scope ) );
+ final GraphManager gm = emf.createEdgeManager( scope );
Id sourceId = new SimpleId( "source" );
Id targetId1 = new SimpleId( "target" );
@@ -1046,7 +1045,7 @@ public abstract class GraphManagerIT {
@Test
public void testMarkTargetEdges() {
- final GraphManager gm = getHelper( emf.createEdgeManager( scope ) );
+ final GraphManager gm = emf.createEdgeManager( scope );
Id sourceId1 = new SimpleId( "source" );
Id sourceId2 = new SimpleId( "source2" );
@@ -1114,7 +1113,7 @@ public abstract class GraphManagerIT {
@Test
public void testMarkSourceEdgesType() {
- final GraphManager gm = getHelper( emf.createEdgeManager( scope ) );
+ final GraphManager gm = emf.createEdgeManager( scope );
Id sourceId = new SimpleId( "source" );
Id targetId1 = new SimpleId( "target" );
@@ -1192,7 +1191,7 @@ public abstract class GraphManagerIT {
@Test
public void testMarkTargetEdgesType() {
- final GraphManager gm = getHelper( emf.createEdgeManager( scope ) );
+ final GraphManager gm = emf.createEdgeManager( scope );
Id sourceId1 = new SimpleId( "source" );
Id sourceId2 = new SimpleId( "source2" );
@@ -1270,7 +1269,7 @@ public abstract class GraphManagerIT {
@Test
public void markSourceNode() {
- final GraphManager gm = getHelper( emf.createEdgeManager( scope ) );
+ final GraphManager gm = emf.createEdgeManager( scope );
Id sourceId = new SimpleId( "source" );
Id targetId1 = new SimpleId( "target" );
@@ -1352,7 +1351,7 @@ public abstract class GraphManagerIT {
@Test
public void markTargetNode() {
- final GraphManager gm = getHelper( emf.createEdgeManager( scope ) );
+ final GraphManager gm = emf.createEdgeManager( scope );
Id sourceId1 = new SimpleId( "source" );
Id sourceId2 = new SimpleId( "source2" );
@@ -1434,7 +1433,7 @@ public abstract class GraphManagerIT {
@Test
public void testWriteReadEdgeTypesSourceTypesPrefix() {
- final GraphManager gm = getHelper( emf.createEdgeManager( scope ) );
+ final GraphManager gm = emf.createEdgeManager( scope );
Id sourceId = new SimpleId( "source" );
Id targetId = new SimpleId( "target" );
@@ -1484,7 +1483,7 @@ public abstract class GraphManagerIT {
public void testSourceSubTypes() {
//now test sub edges
- final GraphManager gm = getHelper( emf.createEdgeManager( scope ) );
+ final GraphManager gm = emf.createEdgeManager( scope );
Id sourceId = new SimpleId( "source" );
Id targetId1target1 = new SimpleId( "type1target1" );
@@ -1533,7 +1532,7 @@ public abstract class GraphManagerIT {
@Test
public void testWriteReadEdgeTypesTargetTypesPrefix() {
- final GraphManager gm = getHelper( emf.createEdgeManager( scope ) );
+ final GraphManager gm = emf.createEdgeManager( scope );
Id targetId = new SimpleId( "target" );
Id sourceId = new SimpleId( "source" );
@@ -1583,7 +1582,7 @@ public abstract class GraphManagerIT {
public void testTargetSubTypes() {
//now test sub edges
- final GraphManager gm = getHelper( emf.createEdgeManager( scope ) );
+ final GraphManager gm = emf.createEdgeManager( scope );
Id targetId = new SimpleId( "target" );
Id sourceId1target1 = new SimpleId( "type1source1" );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/93bacf51/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/StorageGraphManagerIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/StorageGraphManagerIT.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/StorageGraphManagerIT.java
deleted file mode 100644
index 84825aa..0000000
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/StorageGraphManagerIT.java
+++ /dev/null
@@ -1,240 +0,0 @@
-/*
- *
- * * Licensed to the Apache Software Foundation (ASF) under one
- * * or more contributor license agreements. See the NOTICE file
- * * distributed with this work for additional information
- * * regarding copyright ownership. The ASF licenses this file
- * * to you under the Apache License, Version 2.0 (the
- * * "License"); you may not use this file except in compliance
- * * with the License. You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing,
- * * software distributed under the License is distributed on an
- * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * * KIND, either express or implied. See the License for the
- * * specific language governing permissions and limitations
- * * under the License.
- *
- */
-package org.apache.usergrid.persistence.graph;
-
-
-import java.util.concurrent.atomic.AtomicInteger;
-
-import org.junit.Before;
-import org.junit.runner.RunWith;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.usergrid.persistence.core.test.ITRunner;
-import org.apache.usergrid.persistence.core.test.UseModules;
-import org.apache.usergrid.persistence.graph.guice.TestGraphModule;
-import org.apache.usergrid.persistence.graph.impl.GraphManagerImpl;
-import org.apache.usergrid.persistence.model.entity.Id;
-
-import rx.Observable;
-import rx.Observer;
-
-
-/**
- * Integration test that will block reads until all post processing has completed. This ensures once our data has made
- * it to the permanent storage. Tests that our view is immediately consistent to our users, even if we have yet to
- * perform background processing
- */
-@RunWith( ITRunner.class )
-@UseModules( { TestGraphModule.class } )
-public class StorageGraphManagerIT extends GraphManagerIT {
-
- private static final Logger LOG = LoggerFactory.getLogger( StorageGraphManagerIT.class );
-
-
- @Before
- public void setup(){
- }
-
-
-
- @Override
- protected GraphManager getHelper( final GraphManager gm ) {
-
-
- final StorageGraphTestHelper helper = new StorageGraphTestHelper( gm );
-
- GraphManagerImpl gmi = ( GraphManagerImpl ) gm;
-
-
- Observer<Integer> subscriber = new Observer<Integer>() {
- @Override
- public void onCompleted() {
- helper.complete();
- }
-
-
- @Override
- public void onError( final Throwable e ) {
- helper.complete();
- helper.error();
- }
-
-
- @Override
- public void onNext( final Integer integer ) {
- //no op
- }
- };
-
- gmi.setEdgeDeleteSubcriber(subscriber );
-// gmi.setEdgeWriteSubcriber( subscriber );
- gmi.setNodeDelete( subscriber );
-
- return helper;
- }
-
-
- /**
- * Doesn't wait for the async process to happen before returning. Simply executes and immediately returns.
- */
- public static class StorageGraphTestHelper implements GraphManager {
-
- private final GraphManager graphManager;
- private final AtomicInteger completeInvocations = new AtomicInteger( 0 );
- private final AtomicInteger errorInvocations = new AtomicInteger( 0 );
- private final Object mutex = new Object();
-
-
- public StorageGraphTestHelper( final GraphManager graphManager ) {
- this.graphManager = graphManager;
- }
-
-
- @Override
- public Observable<Edge> writeEdge( final Edge edge ) {
- return graphManager.writeEdge( edge );
- }
-
-
- @Override
- public Observable<Edge> markEdge( final Edge edge ) {
- waitForComplete();
- return graphManager.markEdge( edge );
- }
-
-
- @Override
- public Observable<Id> markNode( final Id node, final long timestamp ) {
- waitForComplete();
- return graphManager.markNode( node, timestamp );
- }
-
-
- @Override
- public Observable<Edge> loadEdgeVersions( final SearchByEdge edge ) {
- await();
- return graphManager.loadEdgeVersions( edge );
- }
-
-
- @Override
- public Observable<Edge> loadEdgesFromSource( final SearchByEdgeType search ) {
- await();
- return graphManager.loadEdgesFromSource( search );
- }
-
-
- @Override
- public Observable<Edge> loadEdgesToTarget( final SearchByEdgeType search ) {
- await();
- return graphManager.loadEdgesToTarget( search );
- }
-
-
- @Override
- public Observable<Edge> loadEdgesFromSourceByType( final SearchByIdType search ) {
- await();
- return graphManager.loadEdgesFromSourceByType( search );
- }
-
-
- @Override
- public Observable<Edge> loadEdgesToTargetByType( final SearchByIdType search ) {
- await();
- return graphManager.loadEdgesToTargetByType( search );
- }
-
-
- @Override
- public Observable<String> getEdgeTypesFromSource( final SearchEdgeType search ) {
- await();
- return graphManager.getEdgeTypesFromSource( search );
- }
-
-
- @Override
- public Observable<String> getIdTypesFromSource( final SearchIdType search ) {
- await();
- return graphManager.getIdTypesFromSource( search );
- }
-
-
- @Override
- public Observable<String> getEdgeTypesToTarget( final SearchEdgeType search ) {
- await();
- return graphManager.getEdgeTypesToTarget( search );
- }
-
-
- @Override
- public Observable<String> getIdTypesToTarget( final SearchIdType search ) {
- await();
- return graphManager.getIdTypesToTarget( search );
- }
-
-
- public void waitForComplete(){
- LOG.info( "Complete incremented" );
- completeInvocations.incrementAndGet();
- }
-
- public void complete() {
- LOG.info( "Complete decremented" );
- completeInvocations.decrementAndGet();
- tryWake();
- }
-
-
- public void error() {
- LOG.info( "Error incremented" );
- errorInvocations.incrementAndGet();
- tryWake();
- }
-
-
- public void tryWake() {
- synchronized ( mutex ) {
- mutex.notify();
- }
- }
-
-
- /**
- * Away for our invocations to be 0
- */
- public void await() {
- while ( completeInvocations.get() != 0 ) {
-
- LOG.info( "Waiting for more invocations, count is {} ", completeInvocations.get() );
-
- synchronized ( mutex ) {
- try {
- mutex.wait(1000 );
- }
- catch ( InterruptedException e ) {
- //no op
- }
- }
- }
- }
- }
-}
[2/9] incubator-usergrid git commit: First pass at refactoring mark +
sweep
Posted by to...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3e2afe23/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccLogEntrySerializationStrategyImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccLogEntrySerializationStrategyImplTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccLogEntrySerializationStrategyImplTest.java
index 107b2a0..9ed254c 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccLogEntrySerializationStrategyImplTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccLogEntrySerializationStrategyImplTest.java
@@ -20,6 +20,8 @@
package org.apache.usergrid.persistence.collection.serialization.impl;
+import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.UUID;
@@ -43,6 +45,7 @@ import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.entity.SimpleId;
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
+import com.fasterxml.uuid.UUIDComparator;
import com.google.inject.Inject;
import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
@@ -65,15 +68,15 @@ public abstract class MvccLogEntrySerializationStrategyImplTest {
private MvccLogEntrySerializationStrategy logEntryStrategy;
+
@Before
- public void wireLogEntryStrategy(){
+ public void wireLogEntryStrategy() {
logEntryStrategy = getLogEntryStrategy();
}
/**
* Get the log entry strategy from
- * @return
*/
protected abstract MvccLogEntrySerializationStrategy getLogEntryStrategy();
@@ -181,6 +184,131 @@ public abstract class MvccLogEntrySerializationStrategyImplTest {
}
+ @Test
+ public void getReversedEntries() throws ConnectionException {
+
+ final Id applicationId = new SimpleId( "application" );
+
+ ApplicationScope context = new ApplicationScopeImpl( applicationId );
+
+
+ final Id id = new SimpleId( "test" );
+
+ int count = 10;
+
+ final UUID[] versions = new UUID[count];
+ final Stage COMPLETE = Stage.COMPLETE;
+ final MvccLogEntry[] entries = new MvccLogEntry[count];
+
+
+ for ( int i = 0; i < count; i++ ) {
+ versions[i] = UUIDGenerator.newTimeUUID();
+
+ entries[i] = new MvccLogEntryImpl( id, versions[i], COMPLETE, MvccLogEntry.State.COMPLETE );
+ logEntryStrategy.write( context, entries[i] ).execute();
+
+ //Read it back
+
+ MvccLogEntry returned =
+ logEntryStrategy.load( context, Collections.singleton( id ), versions[i] ).getMaxVersion( id );
+
+ assertNotNull( "Returned value should not be null", returned );
+
+ assertEquals( "Returned should equal the saved", entries[i], returned );
+ }
+
+
+ final UUID[] assertVersions = Arrays.copyOf( versions, versions.length );
+
+ Arrays.sort( assertVersions, ( v1, v2 ) -> UUIDComparator.staticCompare( v1, v2 ) * -1 );
+
+ //now do a range scan from the end
+
+ final int half = count / 2;
+
+ final List<MvccLogEntry> results = logEntryStrategy.loadReversed( context, id, versions[0], half );
+
+ assertEquals( half, results.size() );
+
+ for ( int i = 0; i < count / 2; i++ ) {
+ final MvccLogEntry saved = entries[i];
+ final MvccLogEntry returned = results.get( i );
+
+ assertEquals( "Entry was not equal to the saved value", saved, returned );
+ }
+
+
+ //now get the next batch
+ final List<MvccLogEntry> results2 = logEntryStrategy.loadReversed( context, id, versions[half], count );
+
+ assertEquals( half, results2.size() );
+
+ for ( int i = 0; i < half; i++ ) {
+ final MvccLogEntry saved = entries[half + i];
+ final MvccLogEntry returned = results2.get( i );
+
+ assertEquals( "Entry was not equal to the saved value", saved, returned );
+ }
+
+
+ //now delete them all and ensure we get no results back
+ for ( int i = 0; i < count; i++ ) {
+ logEntryStrategy.delete( context, id, versions[i] ).execute();
+ }
+
+ final List<MvccLogEntry> results3 = logEntryStrategy.loadReversed( context, id, null, versions.length );
+
+ assertEquals( "All log entries were deleted", 0, results3.size() );
+ }
+
+
+ @Test
+ public void createAndDeleteEntries() throws ConnectionException {
+
+ final Id applicationId = new SimpleId( "application" );
+
+ ApplicationScope context = new ApplicationScopeImpl( applicationId );
+
+
+ final Id id = new SimpleId( "test" );
+
+
+ final int size = 10;
+
+ final List<MvccLogEntry> savedEntries = new ArrayList<>( size );
+
+ for ( int i = 0; i < size; i++ ) {
+ final UUID version = UUIDGenerator.newTimeUUID();
+ MvccLogEntry saved = new MvccLogEntryImpl( id, version, Stage.COMMITTED, MvccLogEntry.State.COMPLETE );
+ logEntryStrategy.write( context, saved ).execute();
+
+ savedEntries.add( saved );
+ }
+
+ //now test we get them all back
+
+ final List<MvccLogEntry> results = logEntryStrategy.loadReversed( context, id, null, size );
+
+ assertEquals( size, results.size() );
+
+ //assert they're the same
+ for ( int i = 0; i < size; i++ ) {
+ assertEquals( savedEntries.get( i ), results.get( i ) );
+ }
+
+ //now delete them all
+
+ for ( final MvccLogEntry mvccLogEntry : savedEntries ) {
+ logEntryStrategy.delete( context, id, mvccLogEntry.getVersion() ).execute();
+ }
+
+ //now get them back, should be empty
+ final List<MvccLogEntry> emptyResults = logEntryStrategy.loadReversed( context, id, null, size );
+
+ assertEquals( 0, emptyResults.size() );
+ }
+
+
@Test( expected = NullPointerException.class )
public void writeParamsNoContext() throws ConnectionException {
logEntryStrategy.write( null, mock( MvccLogEntry.class ) );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3e2afe23/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/util/LogEntryMock.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/util/LogEntryMock.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/util/LogEntryMock.java
index 93de9d4..cd6ad3d 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/util/LogEntryMock.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/util/LogEntryMock.java
@@ -20,15 +20,11 @@ package org.apache.usergrid.persistence.collection.util;/*
import java.util.ArrayList;
import java.util.Collection;
-import java.util.Comparator;
import java.util.Iterator;
import java.util.List;
import java.util.TreeMap;
import java.util.UUID;
-import org.mockito.invocation.InvocationOnMock;
-import org.mockito.stubbing.Answer;
-
import org.apache.usergrid.persistence.collection.MvccLogEntry;
import org.apache.usergrid.persistence.collection.mvcc.entity.Stage;
import org.apache.usergrid.persistence.collection.mvcc.entity.impl.MvccLogEntryImpl;
@@ -50,7 +46,11 @@ import static org.mockito.Mockito.when;
public class LogEntryMock {
- private final TreeMap<UUID, MvccLogEntry> entries = new TreeMap<>(ReversedUUIDComparator.INSTANCE);
+ private final TreeMap<UUID, MvccLogEntry> reversedEntries =
+ new TreeMap<>( ( o1, o2 ) -> UUIDComparator.staticCompare( o1, o2 ) * -1 );
+
+ private final TreeMap<UUID, MvccLogEntry> entries =
+ new TreeMap<>( ( o1, o2 ) -> UUIDComparator.staticCompare( o1, o2 ) );
private final Id entityId;
@@ -61,78 +61,92 @@ public class LogEntryMock {
* @param entityId The entity Id to use
* @param versions The versions to use
*/
- private LogEntryMock(final Id entityId, final List<UUID> versions ) {
+ private LogEntryMock( final Id entityId, final List<UUID> versions ) {
this.entityId = entityId;
- for ( UUID version: versions) {
- entries.put( version, new MvccLogEntryImpl( entityId, version, Stage.ACTIVE, MvccLogEntry.State.COMPLETE ) );
+ for ( UUID version : versions ) {
+ final MvccLogEntry mvccLogEntry =
+ new MvccLogEntryImpl( entityId, version, Stage.ACTIVE, MvccLogEntry.State.COMPLETE );
+ reversedEntries.put( version, mvccLogEntry );
+ entries.put( version, mvccLogEntry );
}
}
/**
* Init the mock with the given data structure
+ *
* @param logEntrySerializationStrategy The strategy to moc
- * @param scope
- * @throws ConnectionException
*/
- private void initMock( final MvccLogEntrySerializationStrategy logEntrySerializationStrategy, final ApplicationScope scope )
+ private void initMock( final MvccLogEntrySerializationStrategy logEntrySerializationStrategy,
+ final ApplicationScope scope )
- throws ConnectionException {
+ throws ConnectionException {
//wire up the mocks
- when(logEntrySerializationStrategy.load( same( scope ), same( entityId ), any(UUID.class), any(Integer.class) )).thenAnswer( new Answer<List<MvccLogEntry>>() {
-
+ when( logEntrySerializationStrategy
+ .load( same( scope ), same( entityId ), any( UUID.class ), any( Integer.class ) ) ).thenAnswer(
- @Override
- public List<MvccLogEntry> answer( final InvocationOnMock invocation ) throws Throwable {
+ invocation -> {
final UUID startVersion = ( UUID ) invocation.getArguments()[2];
- final int count = (Integer)invocation.getArguments()[3];
+ final int count = ( Integer ) invocation.getArguments()[3];
final List<MvccLogEntry> results = new ArrayList<>( count );
- final Iterator<MvccLogEntry> itr = entries.tailMap( startVersion, true ).values().iterator();
+ final Iterator<MvccLogEntry> itr = reversedEntries.tailMap( startVersion, true ).values().iterator();
- for(int i = 0; i < count && itr.hasNext(); i ++){
+ for ( int i = 0; i < count && itr.hasNext(); i++ ) {
results.add( itr.next() );
}
return results;
- }
- } );
- }
+ } );
- /**
- * Get the entry at the specified index from high to low
- * @param index
- * @return
- */
- public MvccLogEntry getEntryAtIndex(final int index){
+ //mock in reverse
- final Iterator<MvccLogEntry> itr = entries.values().iterator();
+ when( logEntrySerializationStrategy
+ .loadReversed( same( scope ), same( entityId ), any( UUID.class ), any( Integer.class ) ) ).thenAnswer(
- for(int i = 0; i < index; i ++){
- itr.next();
- }
+ invocation -> {
+ final UUID startVersion = ( UUID ) invocation.getArguments()[2];
+ final int count = ( Integer ) invocation.getArguments()[3];
+
+
+ final List<MvccLogEntry> results = new ArrayList<>( count );
+
+ final Iterator<MvccLogEntry> itr;
+
+ if ( startVersion == null ) {
+ itr = entries.values().iterator();
+ }
+ else {
+ itr = entries.tailMap( startVersion, true ).values().iterator();
+ }
+
+ for ( int i = 0; i < count && itr.hasNext(); i++ ) {
+ results.add( itr.next() );
+ }
- return itr.next();
+
+ return results;
+ } );
}
/**
- *
* @param logEntrySerializationStrategy The mock to use
* @param scope The scope to use
* @param entityId The entityId to use
* @param versions The versions to mock
- * @throws ConnectionException
*/
- public static LogEntryMock createLogEntryMock(final MvccLogEntrySerializationStrategy logEntrySerializationStrategy, final ApplicationScope scope,final Id entityId, final List<UUID> versions )
+ public static LogEntryMock createLogEntryMock(
+ final MvccLogEntrySerializationStrategy logEntrySerializationStrategy, final ApplicationScope scope,
+ final Id entityId, final List<UUID> versions )
- throws ConnectionException {
+ throws ConnectionException {
LogEntryMock mock = new LogEntryMock( entityId, versions );
mock.initMock( logEntrySerializationStrategy, scope );
@@ -141,19 +155,12 @@ public class LogEntryMock {
}
- public Collection<MvccLogEntry> getEntries() {
- return entries.values();
+ public Collection<MvccLogEntry> getReversedEntries() {
+ return reversedEntries.values();
}
- private static final class ReversedUUIDComparator implements Comparator<UUID> {
-
- public static final ReversedUUIDComparator INSTANCE = new ReversedUUIDComparator();
-
-
- @Override
- public int compare( final UUID o1, final UUID o2 ) {
- return UUIDComparator.staticCompare( o1, o2 ) * -1;
- }
+ public Collection<MvccLogEntry> getEntries() {
+ return entries.values();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3e2afe23/stack/corepersistence/collection/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/resources/log4j.properties b/stack/corepersistence/collection/src/test/resources/log4j.properties
index acf5c39..7b55cf8 100644
--- a/stack/corepersistence/collection/src/test/resources/log4j.properties
+++ b/stack/corepersistence/collection/src/test/resources/log4j.properties
@@ -33,4 +33,5 @@ log4j.logger.cassandra.db=ERROR
#log4j.logger.org.apache.usergrid=DEBUG
#log4j.logger.org.apache.usergrid.persistence.collection=TRACE
+log4j.logger.org.apache.usergrid.persistence.collection.mvcc.stage.delete.VersionCompact=TRACE
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3e2afe23/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/executor/TaskExecutorFactory.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/executor/TaskExecutorFactory.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/executor/TaskExecutorFactory.java
new file mode 100644
index 0000000..c653458
--- /dev/null
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/executor/TaskExecutorFactory.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence.core.executor;
+
+
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+
+
+/**
+ * A task executor that allows you to submit tasks
+ */
+public class TaskExecutorFactory {
+
+
+ /**
+ * Create a task executor
+ * @param schedulerName
+ * @param maxThreadCount
+ * @param maxQueueSize
+ * @return
+ */
+ public static ThreadPoolExecutor createTaskExecutor( final String schedulerName, final int maxThreadCount,
+ final int maxQueueSize ) {
+
+
+ final BlockingQueue<Runnable> queue = new ArrayBlockingQueue<Runnable>( maxQueueSize );
+
+
+ final MaxSizeThreadPool threadPool = new MaxSizeThreadPool( queue, schedulerName, maxThreadCount );
+
+
+ return threadPool;
+ }
+
+
+ /**
+ * Create a thread pool that will reject work if our audit tasks become overwhelmed
+ */
+ private static final class MaxSizeThreadPool extends ThreadPoolExecutor {
+
+ public MaxSizeThreadPool( final BlockingQueue<Runnable> queue, final String poolName, final int maxPoolSize ) {
+ super( maxPoolSize, maxPoolSize, 30, TimeUnit.SECONDS, queue, new CountingThreadFactory( poolName ) );
+ }
+ }
+
+
+ /**
+ * Thread factory that will name and count threads for easier debugging
+ */
+ private static final class CountingThreadFactory implements ThreadFactory {
+
+ private final AtomicLong threadCounter = new AtomicLong();
+ private final String poolName;
+
+
+ private CountingThreadFactory( final String poolName ) {this.poolName = poolName;}
+
+
+ @Override
+ public Thread newThread( final Runnable r ) {
+ final long newValue = threadCounter.incrementAndGet();
+
+ final String threadName = poolName + "-" + newValue;
+
+ Thread t = new Thread( r, threadName );
+
+ //set it to be a daemon thread so it doesn't block shutdown
+ t.setDaemon( true );
+
+ return t;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3e2afe23/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/rx/RxTaskScheduler.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/rx/RxTaskScheduler.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/rx/RxTaskScheduler.java
index d6cc5e8..f28e190 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/rx/RxTaskScheduler.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/rx/RxTaskScheduler.java
@@ -20,8 +20,6 @@
package org.apache.usergrid.persistence.core.rx;
-import org.apache.usergrid.persistence.core.task.Task;
-
import rx.Scheduler;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3e2afe23/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/task/NamedTaskExecutorImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/task/NamedTaskExecutorImpl.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/task/NamedTaskExecutorImpl.java
deleted file mode 100644
index 9007167..0000000
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/task/NamedTaskExecutorImpl.java
+++ /dev/null
@@ -1,286 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.usergrid.persistence.core.task;
-
-
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Future;
-import java.util.concurrent.RejectedExecutionException;
-import java.util.concurrent.RejectedExecutionHandler;
-import java.util.concurrent.SynchronousQueue;
-import java.util.concurrent.ThreadFactory;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-import java.util.concurrent.atomic.AtomicLong;
-
-import javax.annotation.Nullable;
-
-import org.slf4j.Logger;
-
-import com.google.common.base.Preconditions;
-import com.google.common.util.concurrent.FutureCallback;
-import com.google.common.util.concurrent.Futures;
-import com.google.common.util.concurrent.ListenableFuture;
-import com.google.common.util.concurrent.ListeningExecutorService;
-import com.google.common.util.concurrent.MoreExecutors;
-
-
-/**
- * Implementation of the task executor with a unique name and size
- */
-public class NamedTaskExecutorImpl implements TaskExecutor {
-
- private static final Logger LOG = org.slf4j.LoggerFactory.getLogger( NamedTaskExecutorImpl.class );
-
- private final ListeningExecutorService executorService;
-
- private final String name;
- private final int poolSize;
-
-
- /**
- * @param name The name of this instance of the task executor
- * @param poolSize The size of the pool. This is the number of concurrent tasks that can execute at once.
- * @param queueLength The length of tasks to keep in the queue
- */
- public NamedTaskExecutorImpl( final String name, final int poolSize, final int queueLength ) {
- Preconditions.checkNotNull( name );
- Preconditions.checkArgument( name.length() > 0, "name must have a length" );
- Preconditions.checkArgument( poolSize > -1, "poolSize must be > than -1" );
- Preconditions.checkArgument( queueLength > -1, "queueLength must be 0 or more" );
-
- this.name = name;
- this.poolSize = poolSize;
-
- // The user has chosen to disable asynchronous execution,
- // to create an executor service that will reject all requests
- if ( poolSize == 0 ) {
- executorService = MoreExecutors.listeningDecorator( new RejectingExecutorService());
- }
-
- //queue executions as normal
- else {
- final BlockingQueue<Runnable> queue = queueLength > 0
- ? new ArrayBlockingQueue<Runnable>(queueLength) : new SynchronousQueue<Runnable>();
-
- executorService = MoreExecutors.listeningDecorator( new MaxSizeThreadPool( queue ) );
- }
- }
-
-
- @Override
- public <V> ListenableFuture<V> submit( final Task<V> task ) {
-
- final ListenableFuture<V> future;
-
- try {
- future = executorService.submit( task );
-
- // Log our success or failures for debugging purposes
- Futures.addCallback( future, new TaskFutureCallBack<V>( task ) );
- }
- catch ( RejectedExecutionException ree ) {
- return Futures.immediateFuture( task.rejected());
- }
-
- return future;
- }
-
-
- @Override
- public void shutdown() {
- this.executorService.shutdownNow();
- }
-
-
- /**
- * Callback for when the task succeeds or fails.
- */
- private static final class TaskFutureCallBack<V> implements FutureCallback<V> {
-
- private final Task<V> task;
-
-
- private TaskFutureCallBack( Task<V> task ) {
- this.task = task;
- }
-
-
- @Override
- public void onSuccess( @Nullable final V result ) {
- LOG.debug( "Successfully completed task ", task );
- }
-
-
- @Override
- public void onFailure( final Throwable t ) {
- LOG.error( "Unable to execute task. Exception is ", t );
-
- task.exceptionThrown( t );
- }
- }
-
-
- /**
- * Create a thread pool that will reject work if our audit tasks become overwhelmed
- */
- private final class MaxSizeThreadPool extends ThreadPoolExecutor {
-
- public MaxSizeThreadPool( BlockingQueue<Runnable> queue ) {
-
- super( 1, poolSize, 30, TimeUnit.SECONDS, queue, new CountingThreadFactory( ),
- new RejectedHandler() );
- }
- }
-
-
- /**
- * Thread factory that will name and count threads for easier debugging
- */
- private final class CountingThreadFactory implements ThreadFactory {
-
- private final AtomicLong threadCounter = new AtomicLong();
-
-
- @Override
- public Thread newThread( final Runnable r ) {
- final long newValue = threadCounter.incrementAndGet();
-
- Thread t = new Thread( r, name + "-" + newValue );
-
- t.setDaemon( true );
-
- return t;
- }
- }
-
-
- /**
- * The handler that will handle rejected executions and signal the interface
- */
- private final class RejectedHandler implements RejectedExecutionHandler {
-
-
- @Override
- public void rejectedExecution( final Runnable r, final ThreadPoolExecutor executor ) {
- LOG.warn( "{} task queue full, rejecting task {}", name, r );
-
- throw new RejectedExecutionException( "Unable to run task, queue full" );
- }
-
- }
-
-
- /**
- * Executor implementation that simply rejects all incoming tasks
- */
- private static final class RejectingExecutorService implements ExecutorService{
-
- @Override
- public void shutdown() {
-
- }
-
-
- @Override
- public List<Runnable> shutdownNow() {
- return Collections.EMPTY_LIST;
- }
-
-
- @Override
- public boolean isShutdown() {
- return false;
- }
-
-
- @Override
- public boolean isTerminated() {
- return false;
- }
-
-
- @Override
- public boolean awaitTermination( final long timeout, final TimeUnit unit )
- throws InterruptedException {
- return false;
- }
-
-
- @Override
- public <T> Future<T> submit( final Callable<T> task ) {
- throw new RejectedExecutionException("No Asynchronous tasks allowed");
- }
-
-
- @Override
- public <T> Future<T> submit( final Runnable task, final T result ) {
- throw new RejectedExecutionException("No Asynchronous tasks allowed");
- }
-
-
- @Override
- public Future<?> submit( final Runnable task ) {
- throw new RejectedExecutionException("No Asynchronous tasks allowed");
- }
-
-
- @Override
- public <T> List<Future<T>> invokeAll( final Collection<? extends Callable<T>> tasks )
- throws InterruptedException {
- throw new RejectedExecutionException("No Asynchronous tasks allowed");
- }
-
-
- @Override
- public <T> List<Future<T>> invokeAll( final Collection<? extends Callable<T>> tasks,
- final long timeout, final TimeUnit unit ) throws InterruptedException {
-
- throw new RejectedExecutionException("No Asynchronous tasks allowed");
- }
-
-
- @Override
- public <T> T invokeAny( final Collection<? extends Callable<T>> tasks )
- throws InterruptedException, ExecutionException {
- throw new RejectedExecutionException("No Asynchronous tasks allowed");
- }
-
-
- @Override
- public <T> T invokeAny( final Collection<? extends Callable<T>> tasks, final long timeout,
- final TimeUnit unit ) throws InterruptedException, ExecutionException, TimeoutException {
- throw new RejectedExecutionException("No Asynchronous tasks allowed");
- }
-
-
- @Override
- public void execute( final Runnable command ) {
- throw new RejectedExecutionException("No Asynchronous tasks allowed");
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3e2afe23/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/task/Task.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/task/Task.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/task/Task.java
deleted file mode 100644
index 5582161..0000000
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/task/Task.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.usergrid.persistence.core.task;
-
-
-import java.util.concurrent.Callable;
-
-
-/**
- * The task to execute
- */
-public interface Task<V> extends Callable<V> {
-
-
- /**
- * Invoked when this task throws an uncaught exception.
- * @param throwable
- */
- void exceptionThrown(final Throwable throwable);
-
- /**
- * Invoked when we weren't able to run this task by the the thread attempting to schedule the task.
- * If this task MUST be run immediately, you can invoke the call method from within this event to invoke the
- * task in the scheduling thread. Note that this has performance implications to the user. If you can drop the
- * request and process later (lazy repair for instance ) do so.
- *
- */
- V rejected();
-
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3e2afe23/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/task/TaskExecutor.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/task/TaskExecutor.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/task/TaskExecutor.java
deleted file mode 100644
index 5728d2e..0000000
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/task/TaskExecutor.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.usergrid.persistence.core.task;
-
-
-import com.google.common.util.concurrent.ListenableFuture;
-
-
-/**
- * An interface for execution of tasks
- */
-public interface TaskExecutor {
-
- /**
- * Submit the task asynchronously
- * @param task
- */
- public <V> ListenableFuture<V> submit( Task<V> task );
-
- /**
- * Stop the task executor without waiting for scheduled threads to run
- */
- public void shutdown();
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3e2afe23/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/task/NamedTaskExecutorImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/task/NamedTaskExecutorImplTest.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/task/NamedTaskExecutorImplTest.java
deleted file mode 100644
index 4f95918..0000000
--- a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/task/NamedTaskExecutorImplTest.java
+++ /dev/null
@@ -1,271 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.usergrid.persistence.core.task;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.TimeUnit;
-import org.junit.Test;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertSame;
-
-
-/**
- * Tests for the namedtask execution impl
- */
-public class NamedTaskExecutorImplTest {
-
-
- @Test
- public void jobSuccess() throws InterruptedException {
- final TaskExecutor executor = new NamedTaskExecutorImpl( "jobSuccess", 1, 0 );
-
- final CountDownLatch exceptionLatch = new CountDownLatch( 0 );
- final CountDownLatch rejectedLatch = new CountDownLatch( 0 );
- final CountDownLatch runLatch = new CountDownLatch( 1 );
-
- final Task<Void> task = new TestTask<Void>( exceptionLatch, rejectedLatch, runLatch ) {};
-
- executor.submit( task );
-
-
- runLatch.await( 1000, TimeUnit.MILLISECONDS );
-
- assertEquals( 0l, exceptionLatch.getCount() );
-
- assertEquals( 0l, rejectedLatch.getCount() );
- }
-
-
- @Test
- public void exceptionThrown() throws InterruptedException {
- final TaskExecutor executor = new NamedTaskExecutorImpl( "jobSuccess", 1, 0 );
-
- final CountDownLatch exceptionLatch = new CountDownLatch( 1 );
- final CountDownLatch rejectedLatch = new CountDownLatch( 0 );
- final CountDownLatch runLatch = new CountDownLatch( 1 );
-
- final RuntimeException re = new RuntimeException( "throwing exception" );
-
- final TestTask<Void> task = new TestTask<Void>( exceptionLatch, rejectedLatch, runLatch ) {
- @Override
- public Void call() throws Exception {
- super.call();
- throw re;
- }
- };
-
- executor.submit( task );
-
-
- runLatch.await( 1000, TimeUnit.MILLISECONDS );
- exceptionLatch.await( 1000, TimeUnit.MILLISECONDS );
-
- assertSame( re, task.exceptions.get( 0 ) );
-
-
- assertEquals( 0l, rejectedLatch.getCount() );
- }
-
-
- @Test
- public void noCapacity() throws InterruptedException {
- final TaskExecutor executor = new NamedTaskExecutorImpl( "jobSuccess", 1, 0 );
-
- final CountDownLatch exceptionLatch = new CountDownLatch( 0 );
- final CountDownLatch rejectedLatch = new CountDownLatch( 0 );
- final CountDownLatch runLatch = new CountDownLatch( 1 );
-
-
- final TestTask<Void> task = new TestTask<Void>( exceptionLatch, rejectedLatch, runLatch ) {
- @Override
- public Void call() throws Exception {
- super.call();
-
- //park this thread so it takes up a task and the next is rejected
- final Object mutex = new Object();
-
- synchronized ( mutex ) {
- mutex.wait();
- }
-
- return null;
- }
- };
-
- executor.submit( task );
-
-
- runLatch.await( 1000, TimeUnit.MILLISECONDS );
-
- //now submit the second task
-
-
- final CountDownLatch secondRejectedLatch = new CountDownLatch( 1 );
- final CountDownLatch secondExceptionLatch = new CountDownLatch( 0 );
- final CountDownLatch secondRunLatch = new CountDownLatch( 1 );
-
-
- final TestTask<Void> testTask = new TestTask<Void>( exceptionLatch, rejectedLatch, runLatch ) {};
-
- executor.submit( testTask );
-
-
- secondRejectedLatch.await( 1000, TimeUnit.MILLISECONDS );
-
- //if we get here we've been rejected, just double check we didn't run
-
- assertEquals( 1l, secondRunLatch.getCount() );
- assertEquals( 0l, secondExceptionLatch.getCount() );
- }
-
-
- @Test
- public void noCapacityWithQueue() throws InterruptedException {
-
- final int threadPoolSize = 1;
- final int queueSize = 10;
-
- final TaskExecutor executor = new NamedTaskExecutorImpl( "jobSuccess", threadPoolSize, queueSize );
-
- final CountDownLatch exceptionLatch = new CountDownLatch( 0 );
- final CountDownLatch rejectedLatch = new CountDownLatch( 0 );
- final CountDownLatch runLatch = new CountDownLatch( 1 );
-
- int iterations = threadPoolSize + queueSize;
-
- for(int i = 0; i < iterations; i ++) {
-
- final TestTask<Void> task = new TestTask<Void>( exceptionLatch, rejectedLatch, runLatch ) {
- @Override
- public Void call() throws Exception {
- super.call();
-
- //park this thread so it takes up a task and the next is rejected
- final Object mutex = new Object();
-
- synchronized ( mutex ) {
- mutex.wait();
- }
-
- return null;
- }
- };
- executor.submit( task );
- }
-
-
-
- runLatch.await( 1000, TimeUnit.MILLISECONDS );
-
- //now submit the second task
-
-
- final CountDownLatch secondRejectedLatch = new CountDownLatch( 1 );
- final CountDownLatch secondExceptionLatch = new CountDownLatch( 0 );
- final CountDownLatch secondRunLatch = new CountDownLatch( 1 );
-
-
- final TestTask<Void> testTask = new TestTask<Void>( exceptionLatch, rejectedLatch, runLatch ) {};
-
- executor.submit( testTask );
-
-
- secondRejectedLatch.await( 1000, TimeUnit.MILLISECONDS );
-
- //if we get here we've been rejected, just double check we didn't run
-
- assertEquals( 1l, secondRunLatch.getCount() );
- assertEquals( 0l, secondExceptionLatch.getCount() );
- }
-
-
- @Test
- public void rejectingTaskExecutor() throws InterruptedException {
-
- final int threadPoolSize = 0;
- final int queueSize = 0;
-
- final TaskExecutor executor = new NamedTaskExecutorImpl( "jobSuccess", threadPoolSize, queueSize );
-
- final CountDownLatch exceptionLatch = new CountDownLatch( 0 );
- final CountDownLatch rejectedLatch = new CountDownLatch( 1 );
- final CountDownLatch runLatch = new CountDownLatch( 0 );
-
-
- //now submit the second task
-
-
-
- final TestTask<Void> testTask = new TestTask<Void>( exceptionLatch, rejectedLatch, runLatch ) {};
-
- executor.submit( testTask );
-
-
- //should be immediately rejected
- rejectedLatch.await( 1000, TimeUnit.MILLISECONDS );
-
- //if we get here we've been rejected, just double check we didn't run
-
- assertEquals( 0l, exceptionLatch.getCount() );
- assertEquals( 0l, runLatch.getCount() );
- }
-
-
- private static abstract class TestTask<V> implements Task<V> {
-
- private final List<Throwable> exceptions;
- private final CountDownLatch exceptionLatch;
- private final CountDownLatch rejectedLatch;
- private final CountDownLatch runLatch;
-
-
- private TestTask( final CountDownLatch exceptionLatch, final CountDownLatch rejectedLatch,
- final CountDownLatch runLatch ) {
- this.exceptionLatch = exceptionLatch;
- this.rejectedLatch = rejectedLatch;
- this.runLatch = runLatch;
-
- this.exceptions = new ArrayList<>();
- }
-
-
-
- @Override
- public void exceptionThrown( final Throwable throwable ) {
- this.exceptions.add( throwable );
- exceptionLatch.countDown();
- }
-
-
- @Override
- public V rejected() {
- rejectedLatch.countDown();
- return null;
- }
-
-
- @Override
- public V call() throws Exception {
- runLatch.countDown();
- return null;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3e2afe23/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/GraphFig.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/GraphFig.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/GraphFig.java
index 894e74a..2a153e2 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/GraphFig.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/GraphFig.java
@@ -31,46 +31,46 @@ import org.safehaus.guicyfig.Key;
public interface GraphFig extends GuicyFig {
- public static final String SCAN_PAGE_SIZE = "usergrid.graph.scan.page.size";
+ String SCAN_PAGE_SIZE = "usergrid.graph.scan.page.size";
- public static final String REPAIR_CONCURRENT_SIZE = "usergrid.graph.repair.concurrent.size";
+ String REPAIR_CONCURRENT_SIZE = "usergrid.graph.repair.concurrent.size";
/**
* The size of the shards. This is approximate, and should be set lower than what you would like your max to be
*/
- public static final String SHARD_SIZE = "usergrid.graph.shard.size";
+ String SHARD_SIZE = "usergrid.graph.shard.size";
/**
* Number of shards we can cache.
*/
- public static final String SHARD_CACHE_SIZE = "usergrid.graph.shard.cache.size";
+ String SHARD_CACHE_SIZE = "usergrid.graph.shard.cache.size";
/**
* Get the cache timeout. The local cache will exist for this amount of time max (in millis).
*/
- public static final String SHARD_CACHE_TIMEOUT = "usergrid.graph.shard.cache.timeout";
+ String SHARD_CACHE_TIMEOUT = "usergrid.graph.shard.cache.timeout";
/**
* Number of worker threads to refresh the cache
*/
- public static final String SHARD_CACHE_REFRESH_WORKERS = "usergrid.graph.shard.refresh.worker.count";
+ String SHARD_CACHE_REFRESH_WORKERS = "usergrid.graph.shard.refresh.worker.count";
/**
* The size of the worker count for shard auditing
*/
- public static final String SHARD_AUDIT_QUEUE_SIZE = "usergrid.graph.shard.audit.worker.queue.size";
+ String SHARD_AUDIT_QUEUE_SIZE = "usergrid.graph.shard.audit.worker.queue.size";
/**
* The size of the worker count for shard auditing
*/
- public static final String SHARD_AUDIT_WORKERS = "usergrid.graph.shard.audit.worker.count";
+ String SHARD_AUDIT_WORKERS = "usergrid.graph.shard.audit.worker.count";
- public static final String SHARD_REPAIR_CHANCE = "usergrid.graph.shard.repair.chance";
+ String SHARD_REPAIR_CHANCE = "usergrid.graph.shard.repair.chance";
/**
@@ -80,14 +80,14 @@ public interface GraphFig extends GuicyFig {
* Note that you should also pad this for node clock drift. A good value for this would be 2x the shard cache
* timeout + 30 seconds, assuming you have NTP and allow a max drift of 30 seconds
*/
- public static final String SHARD_MIN_DELTA = "usergrid.graph.shard.min.delta";
+ String SHARD_MIN_DELTA = "usergrid.graph.shard.min.delta";
- public static final String COUNTER_WRITE_FLUSH_COUNT = "usergrid.graph.shard.counter.beginFlush.count";
+ String COUNTER_WRITE_FLUSH_COUNT = "usergrid.graph.shard.counter.beginFlush.count";
- public static final String COUNTER_WRITE_FLUSH_INTERVAL = "usergrid.graph.shard.counter.beginFlush.interval";
+ String COUNTER_WRITE_FLUSH_INTERVAL = "usergrid.graph.shard.counter.beginFlush.interval";
- public static final String COUNTER_WRITE_FLUSH_QUEUE_SIZE = "usergrid.graph.shard.counter.queue.size";
+ String COUNTER_WRITE_FLUSH_QUEUE_SIZE = "usergrid.graph.shard.counter.queue.size";
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3e2afe23/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/GraphManager.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/GraphManager.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/GraphManager.java
index 4c38c13..987a36c 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/GraphManager.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/GraphManager.java
@@ -64,23 +64,23 @@ public interface GraphManager extends CPManager {
/**
- * @param edge The edge to delete
+ * @param edge Mark the edge as deleted in the graph
*
*
* EdgeDelete the edge. Implementation should also delete the incoming (reversed) edge. Only deletes the specific version
*/
- Observable<Edge> deleteEdge( Edge edge );
+ Observable<Edge> markEdge( Edge edge );
/**
*
- * Remove the node from the graph.
+ * Mark the node as removed from the graph.
*
* @param node The node to remove
* @param timestamp The timestamp to apply the delete operation. Any edges connected to this node with a timestmap
* <= the specified time will be removed from the graph
* @return
*/
- Observable<Id> deleteNode(Id node, long timestamp);
+ Observable<Id> markNode( Id node, long timestamp );
/**
* Get all versions of this edge where versions <= max version
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3e2afe23/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/GraphManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/GraphManagerFactory.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/GraphManagerFactory.java
index eb49711..d73f767 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/GraphManagerFactory.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/GraphManagerFactory.java
@@ -36,7 +36,7 @@ public interface GraphManagerFactory
*
* @param collectionScope The context to use when creating the graph manager
*/
- public GraphManager createEdgeManager( ApplicationScope collectionScope );
+ GraphManager createEdgeManager( ApplicationScope collectionScope );
void invalidate();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3e2afe23/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/SearchByEdge.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/SearchByEdge.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/SearchByEdge.java
index 114440f..38f62b5 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/SearchByEdge.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/SearchByEdge.java
@@ -59,6 +59,12 @@ public interface SearchByEdge {
long getMaxTimestamp();
/**
+ * Return true if we should filter edges marked for deletion
+ * @return
+ */
+ boolean filterMarked();
+
+ /**
* The optional start parameter. All edges emitted with be > the specified start edge.
* This is useful for paging. Simply use the last value returned in the previous call in the start parameter
* @return
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3e2afe23/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/SearchByEdgeType.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/SearchByEdgeType.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/SearchByEdgeType.java
index 5e47ae0..f213b00 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/SearchByEdgeType.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/SearchByEdgeType.java
@@ -65,6 +65,12 @@ public interface SearchByEdgeType {
*/
Order getOrder();
+ /**
+ * Return true to filter marked edges from the results
+ * @return
+ */
+ boolean filterMarked();
+
/**
* Options for ordering. By default, we want to perform descending for common use cases and read speed. This is our our data
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3e2afe23/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/guice/GraphModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/guice/GraphModule.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/guice/GraphModule.java
index 6cdaef0..4b628d1 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/guice/GraphModule.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/guice/GraphModule.java
@@ -28,8 +28,6 @@ import org.apache.usergrid.persistence.core.migration.data.MigrationPlugin;
import org.apache.usergrid.persistence.core.migration.data.MigrationRelationship;
import org.apache.usergrid.persistence.core.migration.data.VersionedMigrationSet;
import org.apache.usergrid.persistence.core.migration.schema.Migration;
-import org.apache.usergrid.persistence.core.task.NamedTaskExecutorImpl;
-import org.apache.usergrid.persistence.core.task.TaskExecutor;
import org.apache.usergrid.persistence.graph.GraphFig;
import org.apache.usergrid.persistence.graph.GraphManagerFactory;
import org.apache.usergrid.persistence.graph.impl.stage.EdgeDeleteListener;
@@ -196,15 +194,6 @@ public abstract class GraphModule extends AbstractModule {
}
- @Inject
- @Singleton
- @Provides
- @GraphTaskExecutor
- public TaskExecutor graphTaskExecutor( final GraphFig graphFig ) {
- return new NamedTaskExecutorImpl( "graphTaskExecutor", graphFig.getShardAuditWorkerCount(),
- graphFig.getShardAuditWorkerQueueSize() );
- }
-
/**
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3e2afe23/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java
index 9c0c62d..9a8a00f 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java
@@ -120,12 +120,9 @@ public class GraphManagerImpl implements GraphManager {
@Inject
public GraphManagerImpl( final EdgeMetadataSerialization edgeMetadataSerialization,
final EdgeSerialization storageEdgeSerialization,
- final NodeSerialization nodeSerialization,
- final GraphFig graphFig,
- final EdgeDeleteListener edgeDeleteListener,
- final NodeDeleteListener nodeDeleteListener,
- final ApplicationScope scope,
- MetricsFactory metricsFactory) {
+ final NodeSerialization nodeSerialization, final GraphFig graphFig,
+ final EdgeDeleteListener edgeDeleteListener, final NodeDeleteListener nodeDeleteListener,
+ final ApplicationScope scope, MetricsFactory metricsFactory ) {
ValidationUtils.validateApplicationScope( scope );
@@ -146,36 +143,34 @@ public class GraphManagerImpl implements GraphManager {
this.edgeDeleteSubcriber = MetricSubscriber.INSTANCE;
this.nodeDelete = MetricSubscriber.INSTANCE;
- this.writeEdgeMeter = metricsFactory.getMeter(GraphManagerImpl.class, "write.edge.meter");
- this.writeEdgeTimer = metricsFactory.getTimer(GraphManagerImpl.class, "write.edge.timer");
- this.deleteEdgeMeter = metricsFactory.getMeter(GraphManagerImpl.class, "delete.edge.meter");
- this.deleteEdgeTimer = metricsFactory.getTimer(GraphManagerImpl.class, "delete.edge.timer");
- this.deleteNodeMeter = metricsFactory.getMeter(GraphManagerImpl.class, "delete.node.meter");
- this.deleteNodeTimer = metricsFactory.getTimer(GraphManagerImpl.class, "delete.node.timer");
- this.loadEdgesFromSourceMeter = metricsFactory.getMeter(GraphManagerImpl.class, "load.from.meter");
- this.loadEdgesFromSourceTimer = metricsFactory.getTimer(GraphManagerImpl.class, "load.from.timer");
- this.loadEdgesToTargetMeter = metricsFactory.getMeter(GraphManagerImpl.class, "load.to.meter");
- this.loadEdgesToTargetTimer = metricsFactory.getTimer(GraphManagerImpl.class, "load.to.timer");
- this.loadEdgesVersionsMeter = metricsFactory.getMeter(GraphManagerImpl.class, "load.versions.meter");
- this.loadEdgesVersionsTimer = metricsFactory.getTimer(GraphManagerImpl.class,"load.versions.timer");
- this.loadEdgesFromSourceByTypeMeter = metricsFactory.getMeter(GraphManagerImpl.class, "load.from.type.meter");
- this.loadEdgesFromSourceByTypeTimer = metricsFactory.getTimer(GraphManagerImpl.class, "load.from.type.timer");
- this.loadEdgesToTargetByTypeMeter = metricsFactory.getMeter(GraphManagerImpl.class, "load.to.type.meter");
- this.loadEdgesToTargetByTypeTimer = metricsFactory.getTimer(GraphManagerImpl.class, "load.to.type.timer");
-
- this.getEdgeTypesFromSourceTimer = metricsFactory.getTimer(GraphManagerImpl.class,"get.edge.from.timer");
- this.getEdgeTypesFromSourceMeter = metricsFactory.getMeter(GraphManagerImpl.class, "get.edge.from.meter");
-
- this.getIdTypesFromSourceTimer = metricsFactory.getTimer(GraphManagerImpl.class,"get.idtype.from.timer");
- this.getIdTypesFromSourceMeter = metricsFactory.getMeter(GraphManagerImpl.class, "get.idtype.from.meter");
-
- this.getEdgeTypesToTargetTimer = metricsFactory.getTimer(GraphManagerImpl.class,"get.edge.to.timer");
- this.getEdgeTypesToTargetMeter = metricsFactory.getMeter(GraphManagerImpl.class, "get.edge.to.meter");
-
- this.getIdTypesToTargetTimer = metricsFactory.getTimer(GraphManagerImpl.class, "get.idtype.to.timer");
- this.getIdTypesToTargetMeter = metricsFactory.getMeter(GraphManagerImpl.class, "get.idtype.to.meter");
-
-
+ this.writeEdgeMeter = metricsFactory.getMeter( GraphManagerImpl.class, "write.edge.meter" );
+ this.writeEdgeTimer = metricsFactory.getTimer( GraphManagerImpl.class, "write.edge.timer" );
+ this.deleteEdgeMeter = metricsFactory.getMeter( GraphManagerImpl.class, "delete.edge.meter" );
+ this.deleteEdgeTimer = metricsFactory.getTimer( GraphManagerImpl.class, "delete.edge.timer" );
+ this.deleteNodeMeter = metricsFactory.getMeter( GraphManagerImpl.class, "delete.node.meter" );
+ this.deleteNodeTimer = metricsFactory.getTimer( GraphManagerImpl.class, "delete.node.timer" );
+ this.loadEdgesFromSourceMeter = metricsFactory.getMeter( GraphManagerImpl.class, "load.from.meter" );
+ this.loadEdgesFromSourceTimer = metricsFactory.getTimer( GraphManagerImpl.class, "load.from.timer" );
+ this.loadEdgesToTargetMeter = metricsFactory.getMeter( GraphManagerImpl.class, "load.to.meter" );
+ this.loadEdgesToTargetTimer = metricsFactory.getTimer( GraphManagerImpl.class, "load.to.timer" );
+ this.loadEdgesVersionsMeter = metricsFactory.getMeter( GraphManagerImpl.class, "load.versions.meter" );
+ this.loadEdgesVersionsTimer = metricsFactory.getTimer( GraphManagerImpl.class, "load.versions.timer" );
+ this.loadEdgesFromSourceByTypeMeter = metricsFactory.getMeter( GraphManagerImpl.class, "load.from.type.meter" );
+ this.loadEdgesFromSourceByTypeTimer = metricsFactory.getTimer( GraphManagerImpl.class, "load.from.type.timer" );
+ this.loadEdgesToTargetByTypeMeter = metricsFactory.getMeter( GraphManagerImpl.class, "load.to.type.meter" );
+ this.loadEdgesToTargetByTypeTimer = metricsFactory.getTimer( GraphManagerImpl.class, "load.to.type.timer" );
+
+ this.getEdgeTypesFromSourceTimer = metricsFactory.getTimer( GraphManagerImpl.class, "get.edge.from.timer" );
+ this.getEdgeTypesFromSourceMeter = metricsFactory.getMeter( GraphManagerImpl.class, "get.edge.from.meter" );
+
+ this.getIdTypesFromSourceTimer = metricsFactory.getTimer( GraphManagerImpl.class, "get.idtype.from.timer" );
+ this.getIdTypesFromSourceMeter = metricsFactory.getMeter( GraphManagerImpl.class, "get.idtype.from.meter" );
+
+ this.getEdgeTypesToTargetTimer = metricsFactory.getTimer( GraphManagerImpl.class, "get.edge.to.timer" );
+ this.getEdgeTypesToTargetMeter = metricsFactory.getMeter( GraphManagerImpl.class, "get.edge.to.meter" );
+
+ this.getIdTypesToTargetTimer = metricsFactory.getTimer( GraphManagerImpl.class, "get.idtype.to.timer" );
+ this.getIdTypesToTargetMeter = metricsFactory.getMeter( GraphManagerImpl.class, "get.idtype.to.meter" );
}
@@ -209,41 +204,39 @@ public class GraphManagerImpl implements GraphManager {
return edge;
}
- } )
- .doOnEach(new Action1<Notification<? super Edge>>() {
- @Override
- public void call(Notification<? super Edge> notification) {
- meter.mark();
- }
- })
- .doOnCompleted(new Action0() {
- @Override
- public void call() {
- timer.stop();
- }
- });
+ } ).doOnEach( new Action1<Notification<? super Edge>>() {
+ @Override
+ public void call( Notification<? super Edge> notification ) {
+ meter.mark();
+ }
+ } ).doOnCompleted( new Action0() {
+ @Override
+ public void call() {
+ timer.stop();
+ }
+ } );
}
@Override
- public Observable<Edge> deleteEdge( final Edge edge ) {
- GraphValidation.validateEdge(edge);
+ public Observable<Edge> markEdge( final Edge edge ) {
+ GraphValidation.validateEdge( edge );
- final MarkedEdge markedEdge = new SimpleMarkedEdge(edge, true);
+ final MarkedEdge markedEdge = new SimpleMarkedEdge( edge, true );
final Timer.Context timer = deleteEdgeTimer.time();
final Meter meter = deleteEdgeMeter;
- return Observable.just(markedEdge).map(new Func1<MarkedEdge, Edge>() {
+ return Observable.just( markedEdge ).map( new Func1<MarkedEdge, Edge>() {
@Override
- public Edge call(final MarkedEdge edge) {
+ public Edge call( final MarkedEdge edge ) {
final UUID timestamp = UUIDGenerator.newTimeUUID();
- final MutationBatch edgeMutation = storageEdgeSerialization.writeEdge(scope, edge, timestamp);
+ final MutationBatch edgeMutation = storageEdgeSerialization.writeEdge( scope, edge, timestamp );
- LOG.debug("Marking edge {} as deleted to commit log", edge);
+ LOG.debug( "Marking edge {} as deleted to commit log", edge );
try {
edgeMutation.execute();
}
@@ -254,73 +247,50 @@ public class GraphManagerImpl implements GraphManager {
//HystrixCassandra.async( edgeDeleteListener.receive( scope, markedEdge,
// timestamp )).subscribeOn( Schedulers.io() ).subscribe( edgeDeleteSubcriber );
- edgeDeleteListener.receive(scope, markedEdge, timestamp).subscribeOn(Schedulers.io())
- .subscribe(edgeDeleteSubcriber);
+ edgeDeleteListener.receive( scope, markedEdge, timestamp ).subscribeOn( Schedulers.io() )
+ .subscribe( edgeDeleteSubcriber );
return edge;
}
- })
- .doOnEach(new Action1<Notification<? super Edge>>() {
- @Override
- public void call(Notification<? super Edge> notification) {
- meter.mark();
- }
- })
- .doOnCompleted(new Action0() {
- @Override
- public void call() {
- timer.stop();
- }
- });
+ } ).doOnEach( notification -> {
+ meter.mark();
+ } ).doOnCompleted( () -> timer.stop() );
}
@Override
- public Observable<Id> deleteNode( final Id node, final long timestamp ) {
+ public Observable<Id> markNode( final Id node, final long timestamp ) {
final Timer.Context timer = deleteNodeTimer.time();
final Meter meter = deleteNodeMeter;
- return Observable.just( node ).map( new Func1<Id, Id>() {
- @Override
- public Id call( final Id id ) {
+ return Observable.just( node ).map( id -> {
- //mark the node as deleted
+ //mark the node as deleted
- final UUID eventTimestamp = UUIDGenerator.newTimeUUID();
+ final UUID eventTimestamp = UUIDGenerator.newTimeUUID();
- final MutationBatch nodeMutation = nodeSerialization.mark( scope, id, timestamp );
+ final MutationBatch nodeMutation = nodeSerialization.mark( scope, id, timestamp );
- LOG.debug( "Marking node {} as deleted to node mark", node );
- try {
- nodeMutation.execute();
- }
- catch ( ConnectionException e ) {
- throw new RuntimeException( "Unable to execute mutation", e );
- }
+ LOG.debug( "Marking node {} as deleted to node mark", node );
+ try {
+ nodeMutation.execute();
+ }
+ catch ( ConnectionException e ) {
+ throw new RuntimeException( "Unable to execute mutation", e );
+ }
- //HystrixCassandra.async(nodeDeleteListener.receive(scope, id, eventTimestamp )).subscribeOn(
- // Schedulers.io() ).subscribe( nodeDelete );
- nodeDeleteListener.receive( scope, id, eventTimestamp ).subscribeOn( Schedulers.io() )
- .subscribe( nodeDelete );
+ //HystrixCassandra.async(nodeDeleteListener.receive(scope, id, eventTimestamp )).subscribeOn(
+ // Schedulers.io() ).subscribe( nodeDelete );
+ nodeDeleteListener.receive( scope, id, eventTimestamp ).subscribeOn( Schedulers.io() )
+ .subscribe( nodeDelete );
- return id;
- }
- } )
- .doOnEach(new Action1<Notification<? super Id>>() {
- @Override
- public void call(Notification<? super Id> notification) {
- meter.mark();
- }
- })
- .doOnCompleted(new Action0() {
- @Override
- public void call() {
- timer.stop();
- }
- });
+ return id;
+ } ).doOnEach( notification -> {
+ meter.mark();
+ } ).doOnCompleted( () -> timer.stop() );
}
@@ -333,20 +303,11 @@ public class GraphManagerImpl implements GraphManager {
protected Iterator<MarkedEdge> getIterator() {
return storageEdgeSerialization.getEdgeVersions( scope, searchByEdge );
}
- } ).buffer( graphFig.getScanPageSize() ).flatMap(new EdgeBufferFilter(searchByEdge.getMaxTimestamp()))
- .cast(Edge.class)
- .doOnEach(new Action1<Notification<? super Edge>>() {
- @Override
- public void call(Notification<? super Edge> notification) {
- meter.mark();
- }
- })
- .doOnCompleted(new Action0() {
- @Override
- public void call() {
- timer.stop();
- }
- });
+ } ).buffer( graphFig.getScanPageSize() )
+ .flatMap( new EdgeBufferFilter( searchByEdge.getMaxTimestamp(), searchByEdge.filterMarked() ) )
+ .cast( Edge.class ).doOnEach( notification -> {
+ meter.mark();
+ } ).doOnCompleted( () -> timer.stop() );
}
@@ -359,20 +320,11 @@ public class GraphManagerImpl implements GraphManager {
protected Iterator<MarkedEdge> getIterator() {
return storageEdgeSerialization.getEdgesFromSource( scope, search );
}
- } ).buffer( graphFig.getScanPageSize() ).flatMap(new EdgeBufferFilter(search.getMaxTimestamp()))
- .cast(Edge.class)
- .doOnEach(new Action1<Notification<? super Edge>>() {
- @Override
- public void call(Notification<? super Edge> notification) {
- meter.mark();
- }
- })
- .doOnCompleted(new Action0() {
- @Override
- public void call() {
- timer.stop();
- }
- });
+ } ).buffer( graphFig.getScanPageSize() )
+ .flatMap( new EdgeBufferFilter( search.getMaxTimestamp(), search.filterMarked() ) ).cast( Edge.class )
+ .doOnEach( notification -> {
+ meter.mark();
+ } ).doOnCompleted( () -> timer.stop() );
}
@@ -385,20 +337,11 @@ public class GraphManagerImpl implements GraphManager {
protected Iterator<MarkedEdge> getIterator() {
return storageEdgeSerialization.getEdgesToTarget( scope, search );
}
- } ).buffer( graphFig.getScanPageSize() ).flatMap(new EdgeBufferFilter(search.getMaxTimestamp()))
- .cast(Edge.class)
- .doOnEach(new Action1<Notification<? super Edge>>() {
- @Override
- public void call(Notification<? super Edge> notification) {
- meter.mark();
- }
- })
- .doOnCompleted(new Action0() {
- @Override
- public void call() {
- timer.stop();
- }
- });
+ } ).buffer( graphFig.getScanPageSize() )
+ .flatMap( new EdgeBufferFilter( search.getMaxTimestamp(), search.filterMarked() ) ).cast( Edge.class )
+ .doOnEach( notification -> {
+ meter.mark();
+ } ).doOnCompleted( () -> timer.stop() );
}
@@ -411,21 +354,12 @@ public class GraphManagerImpl implements GraphManager {
protected Iterator<MarkedEdge> getIterator() {
return storageEdgeSerialization.getEdgesFromSourceByTargetType( scope, search );
}
- } ).buffer( graphFig.getScanPageSize() ).flatMap(new EdgeBufferFilter(search.getMaxTimestamp()))
+ } ).buffer( graphFig.getScanPageSize() )
+ .flatMap( new EdgeBufferFilter( search.getMaxTimestamp(), search.filterMarked() ) )
- .cast(Edge.class)
- .doOnEach(new Action1<Notification<? super Edge>>() {
- @Override
- public void call(Notification<? super Edge> notification) {
- meter.mark();
- }
- })
- .doOnCompleted(new Action0() {
- @Override
- public void call() {
- timer.stop();
- }
- });
+ .cast( Edge.class ).doOnEach( notification -> {
+ meter.mark();
+ } ).doOnCompleted( () -> timer.stop() );
}
@@ -438,20 +372,11 @@ public class GraphManagerImpl implements GraphManager {
protected Iterator<MarkedEdge> getIterator() {
return storageEdgeSerialization.getEdgesToTargetBySourceType( scope, search );
}
- } ).buffer( graphFig.getScanPageSize() ).flatMap(new EdgeBufferFilter(search.getMaxTimestamp()))
- .cast(Edge.class)
- .doOnEach(new Action1<Notification<? super Edge>>() {
- @Override
- public void call(Notification<? super Edge> notification) {
- meter.mark();
- }
- })
- .doOnCompleted(new Action0() {
- @Override
- public void call() {
- timer.stop();
- }
- });
+ } ).buffer( graphFig.getScanPageSize() )
+ .flatMap( new EdgeBufferFilter( search.getMaxTimestamp(), search.filterMarked() ) ).cast( Edge.class )
+ .doOnEach( notification -> {
+ meter.mark();
+ } ).doOnCompleted( () -> timer.stop() );
}
@@ -464,19 +389,9 @@ public class GraphManagerImpl implements GraphManager {
protected Iterator<String> getIterator() {
return edgeMetadataSerialization.getEdgeTypesFromSource( scope, search );
}
- } )
- .doOnEach(new Action1<Notification<? super String>>() {
- @Override
- public void call(Notification<? super String> notification) {
- meter.mark();
- }
- })
- .doOnCompleted(new Action0() {
- @Override
- public void call() {
- timer.stop();
- }
- });
+ } ).doOnEach( notification -> {
+ meter.mark();
+ } ).doOnCompleted( () -> timer.stop() );
}
@@ -489,19 +404,9 @@ public class GraphManagerImpl implements GraphManager {
protected Iterator<String> getIterator() {
return edgeMetadataSerialization.getIdTypesFromSource( scope, search );
}
- } )
- .doOnEach(new Action1<Notification<? super String>>() {
- @Override
- public void call(Notification<? super String> notification) {
- meter.mark();
- }
- })
- .doOnCompleted(new Action0() {
- @Override
- public void call() {
- timer.stop();
- }
- });
+ } ).doOnEach( notification -> {
+ meter.mark();
+ } ).doOnCompleted( () -> timer.stop() );
}
@@ -514,19 +419,9 @@ public class GraphManagerImpl implements GraphManager {
protected Iterator<String> getIterator() {
return edgeMetadataSerialization.getEdgeTypesToTarget( scope, search );
}
- } )
- .doOnEach(new Action1<Notification<? super String>>() {
- @Override
- public void call(Notification<? super String> notification) {
- meter.mark();
- }
- })
- .doOnCompleted(new Action0() {
- @Override
- public void call() {
- timer.stop();
- }
- });
+ } ).doOnEach( notification -> {
+ meter.mark();
+ } ).doOnCompleted( () -> timer.stop() );
}
@@ -539,19 +434,9 @@ public class GraphManagerImpl implements GraphManager {
protected Iterator<String> getIterator() {
return edgeMetadataSerialization.getIdTypesToTarget( scope, search );
}
- } )
- .doOnEach(new Action1<Notification<? super String>>() {
- @Override
- public void call(Notification<? super String> notification) {
- meter.mark();
- }
- })
- .doOnCompleted(new Action0() {
- @Override
- public void call() {
- timer.stop();
- }
- });
+ } ).doOnEach( notification -> {
+ meter.mark();
+ } ).doOnCompleted( () -> timer.stop() );
}
@@ -561,10 +446,12 @@ public class GraphManagerImpl implements GraphManager {
private class EdgeBufferFilter implements Func1<List<MarkedEdge>, Observable<MarkedEdge>> {
private final long maxVersion;
+ private final boolean filterMarked;
- private EdgeBufferFilter( final long maxVersion ) {
+ private EdgeBufferFilter( final long maxVersion, final boolean filterMarked ) {
this.maxVersion = maxVersion;
+ this.filterMarked = filterMarked;
}
@@ -579,57 +466,36 @@ public class GraphManagerImpl implements GraphManager {
public Observable<MarkedEdge> call( final List<MarkedEdge> markedEdges ) {
final Map<Id, Long> markedVersions = nodeSerialization.getMaxVersions( scope, markedEdges );
- return Observable.from( markedEdges ).filter( new EdgeFilter( this.maxVersion, markedVersions ) );
- }
- }
-
-
- /**
- * Filter the returned values based on the max uuid and if it's been marked for deletion or not
- */
- private static class EdgeFilter implements Func1<MarkedEdge, Boolean> {
-
- private final long maxTimestamp;
-
- private final Map<Id, Long> markCache;
-
-
- private EdgeFilter( final long maxTimestamp, Map<Id, Long> markCache ) {
- this.maxTimestamp = maxTimestamp;
- this.markCache = markCache;
- }
-
-
- @Override
- public Boolean call( final MarkedEdge edge ) {
-
+ final long maxTimestamp = maxVersion;
- final long edgeTimestamp = edge.getTimestamp();
+ return Observable.from( markedEdges ).filter( edge -> {
+ final long edgeTimestamp = edge.getTimestamp();
- //our edge needs to not be deleted and have a version that's > max Version
- if ( edge.isDeleted() || Long.compare( edgeTimestamp, maxTimestamp ) > 0 ) {
- return false;
- }
+ //our edge needs to not be deleted and have a version that's > max Version
+ if ( edge.isDeleted() || Long.compare( edgeTimestamp, maxTimestamp ) > 0 ) {
+ return false;
+ }
- final Long sourceTimestamp = markCache.get( edge.getSourceNode() );
+ final Long sourceTimestamp = markedVersions.get( edge.getSourceNode() );
- //the source Id has been marked for deletion. It's version is <= to the marked version for deletion,
- // so we need to discard it
- if ( sourceTimestamp != null && Long.compare( edgeTimestamp, sourceTimestamp ) < 1 ) {
- return false;
- }
+ //the source Id has been marked for deletion. It's version is <= to the marked version for deletion,
+ // so we need to discard it
+ if ( sourceTimestamp != null && Long.compare( edgeTimestamp, sourceTimestamp ) < 1 ) {
+ return false;
+ }
- final Long targetTimestamp = markCache.get( edge.getTargetNode() );
+ final Long targetTimestamp = markedVersions.get( edge.getTargetNode() );
- //the target Id has been marked for deletion. It's version is <= to the marked version for deletion,
- // so we need to discard it
- if ( targetTimestamp != null && Long.compare( edgeTimestamp, targetTimestamp ) < 1 ) {
- return false;
- }
+ //the target Id has been marked for deletion. It's version is <= to the marked version for deletion,
+ // so we need to discard it
+ if ( targetTimestamp != null && Long.compare( edgeTimestamp, targetTimestamp ) < 1 ) {
+ return false;
+ }
- return true;
+ return true;
+ } );
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3e2afe23/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleSearchByEdge.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleSearchByEdge.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleSearchByEdge.java
index a28a0bb..9a23caf 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleSearchByEdge.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleSearchByEdge.java
@@ -33,7 +33,6 @@ import com.google.common.base.Preconditions;
/**
* Simple bean implementation of search by edge
- *
*/
public class SimpleSearchByEdge implements SearchByEdge {
@@ -43,24 +42,44 @@ public class SimpleSearchByEdge implements SearchByEdge {
private final long maxTimestamp;
private final Optional<Edge> last;
private final SearchByEdgeType.Order order;
+ private final boolean filterMarked;
/**
* Create the search modules
+ *
* @param sourceNode The source node of the edge
* @param targetNode The target node of the edge
* @param type The edge type
* @param maxTimestamp The maximum timestamp to seek from
* @param last The value to start seeking from. Must be >= this value
*/
- public SimpleSearchByEdge( final Id sourceNode, final String type, final Id targetNode, final long maxTimestamp, final SearchByEdgeType.Order order, final Optional<Edge> last ) {
+ public SimpleSearchByEdge( final Id sourceNode, final String type, final Id targetNode, final long maxTimestamp,
+ final SearchByEdgeType.Order order, final Optional<Edge> last ) {
+ this( sourceNode, type, targetNode, maxTimestamp, order, last, true );
+ }
+
+
+ /**
+ * Create the search modules
+ *
+ * @param sourceNode The source node of the edge
+ * @param type The edge type
+ * @param targetNode The target node of the edge
+ * @param maxTimestamp The maximum timestamp to seek from
+ * @param last The value to start seeking from. Must be >= this value
+ */
+ public SimpleSearchByEdge( final Id sourceNode, final String type, final Id targetNode, final long maxTimestamp,
+ final SearchByEdgeType.Order order, final Optional<Edge> last,
+ final boolean filterMarked ) {
- ValidationUtils.verifyIdentity(sourceNode);
- ValidationUtils.verifyIdentity(targetNode);
- ValidationUtils.verifyString(type, "type");
- GraphValidation.validateTimestamp(maxTimestamp, "maxTimestamp");
- Preconditions.checkNotNull(order, "order must not be null");
- Preconditions.checkNotNull(last, "last can never be null");
+
+ ValidationUtils.verifyIdentity( sourceNode );
+ ValidationUtils.verifyIdentity( targetNode );
+ ValidationUtils.verifyString( type, "type" );
+ GraphValidation.validateTimestamp( maxTimestamp, "maxTimestamp" );
+ Preconditions.checkNotNull( order, "order must not be null" );
+ Preconditions.checkNotNull( last, "last can never be null" );
this.sourceNode = sourceNode;
@@ -69,6 +88,7 @@ public class SimpleSearchByEdge implements SearchByEdge {
this.maxTimestamp = maxTimestamp;
this.order = order;
this.last = last;
+ this.filterMarked = filterMarked;
}
@@ -97,6 +117,10 @@ public class SimpleSearchByEdge implements SearchByEdge {
@Override
+ public boolean filterMarked() { return filterMarked; }
+
+
+ @Override
public Optional<Edge> last() {
return last;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3e2afe23/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleSearchByEdgeType.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleSearchByEdgeType.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleSearchByEdgeType.java
index 1687162..9392dbc 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleSearchByEdgeType.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleSearchByEdgeType.java
@@ -41,6 +41,7 @@ public class SimpleSearchByEdgeType implements SearchByEdgeType{
private final long maxTimestamp;
private final Optional<Edge> last;
private final Order order;
+ private final boolean filterMarked;
/**
@@ -55,7 +56,7 @@ public class SimpleSearchByEdgeType implements SearchByEdgeType{
* //TODO, make last an optional
*/
public SimpleSearchByEdgeType( final Id node, final String type, final long maxTimestamp, final Order order, final Edge last ) {
- this(node, type, maxTimestamp, order, Optional.fromNullable(last));
+ this(node, type, maxTimestamp, order, Optional.fromNullable(last), true);
}
@@ -70,7 +71,24 @@ public class SimpleSearchByEdgeType implements SearchByEdgeType{
*
* //TODO, make last an optional
*/
- public SimpleSearchByEdgeType( final Id node, final String type, final long maxTimestamp, final Order order, final Optional<Edge> last ) {
+ public SimpleSearchByEdgeType( final Id node, final String type, final long maxTimestamp, final Order order,
+ final Optional<Edge> last ) {
+ this( node, type, maxTimestamp, order, last, true );
+ }
+
+
+ /**
+ * Create the search modules
+ * @param node The node to search from
+ * @param type The edge type
+ * @param maxTimestamp The maximum timestamp to return
+ * @param order The order order. Descending is most efficient
+ * @param last The value to start seeking from. Must be >= this value
+ * @param filterMarked
+ */
+ public SimpleSearchByEdgeType( final Id node, final String type, final long maxTimestamp, final Order order,
+ final Optional<Edge> last, final boolean filterMarked ) {
+
Preconditions.checkNotNull( order, "order is required");
ValidationUtils.verifyIdentity( node );
@@ -84,6 +102,7 @@ public class SimpleSearchByEdgeType implements SearchByEdgeType{
this.maxTimestamp = maxTimestamp;
this.order = order;
this.last = last;
+ this.filterMarked = filterMarked;
}
@@ -118,6 +137,12 @@ public class SimpleSearchByEdgeType implements SearchByEdgeType{
@Override
+ public boolean filterMarked() {
+ return filterMarked;
+ }
+
+
+ @Override
public boolean equals( final Object o ) {
if ( this == o ) {
return true;
[8/9] incubator-usergrid git commit: Migrated filter to a transform
operation
Posted by to...@apache.org.
Migrated filter to a transform operation
Updated tests
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/62e650bc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/62e650bc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/62e650bc
Branch: refs/heads/two-dot-o-dev
Commit: 62e650bc13e50b029f649df7503c1e2b604648ed
Parents: 93bacf5
Author: Todd Nine <tn...@apigee.com>
Authored: Sun May 10 04:58:24 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Sun May 10 04:59:22 2015 -0600
----------------------------------------------------------------------
.../graph/impl/GraphManagerImpl.java | 144 ++--
.../graph/impl/SimpleSearchByIdType.java | 8 +
.../persistence/graph/GraphManagerIT.java | 664 ++++++++++++++++---
.../graph/test/util/EdgeTestUtils.java | 27 +
4 files changed, 672 insertions(+), 171 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/62e650bc/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java
index 06cb5a1..e4ce4fd 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java
@@ -51,7 +51,6 @@ import org.apache.usergrid.persistence.graph.serialization.util.GraphValidation;
import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
-import com.codahale.metrics.Meter;
import com.codahale.metrics.Timer;
import com.google.common.base.Optional;
import com.google.common.base.Preconditions;
@@ -60,7 +59,6 @@ import com.netflix.astyanax.MutationBatch;
import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
import rx.Observable;
-import rx.functions.Func1;
/**
@@ -293,7 +291,7 @@ public class GraphManagerImpl implements GraphManager {
return storageEdgeSerialization.getEdgeVersions( scope, searchByEdge );
}
} ).buffer( graphFig.getScanPageSize() )
- .flatMap( new EdgeBufferFilter( searchByEdge.getMaxTimestamp(), searchByEdge.filterMarked() ) );
+ .compose( new EdgeBufferFilter( searchByEdge.getMaxTimestamp(), searchByEdge.filterMarked() ) );
return ObservableTimer.time( edges, loadEdgesVersionsTimer );
}
@@ -308,7 +306,7 @@ public class GraphManagerImpl implements GraphManager {
return storageEdgeSerialization.getEdgesFromSource( scope, search );
}
} ).buffer( graphFig.getScanPageSize() )
- .flatMap( new EdgeBufferFilter( search.getMaxTimestamp(), search.filterMarked() ) );
+ .compose( new EdgeBufferFilter( search.getMaxTimestamp(), search.filterMarked() ) );
return ObservableTimer.time( edges, loadEdgesFromSourceTimer );
}
@@ -323,7 +321,7 @@ public class GraphManagerImpl implements GraphManager {
return storageEdgeSerialization.getEdgesToTarget( scope, search );
}
} ).buffer( graphFig.getScanPageSize() )
- .flatMap( new EdgeBufferFilter( search.getMaxTimestamp(), search.filterMarked() ) );
+ .compose( new EdgeBufferFilter( search.getMaxTimestamp(), search.filterMarked() ) );
return ObservableTimer.time( edges, loadEdgesToTargetTimer );
@@ -334,14 +332,14 @@ public class GraphManagerImpl implements GraphManager {
public Observable<Edge> loadEdgesFromSourceByType( final SearchByIdType search ) {
final Observable<Edge> edges =
Observable.create( new ObservableIterator<MarkedEdge>( "getEdgeTypesFromSource" ) {
- @Override
- protected Iterator<MarkedEdge> getIterator() {
- return storageEdgeSerialization.getEdgesFromSourceByTargetType( scope, search );
- }
- } ).buffer( graphFig.getScanPageSize() )
- .flatMap( new EdgeBufferFilter( search.getMaxTimestamp(), search.filterMarked() ) );
+ @Override
+ protected Iterator<MarkedEdge> getIterator() {
+ return storageEdgeSerialization.getEdgesFromSourceByTargetType( scope, search );
+ }
+ } ).buffer( graphFig.getScanPageSize() )
+ .compose( new EdgeBufferFilter( search.getMaxTimestamp(), search.filterMarked() ) );
- return ObservableTimer.time( edges, loadEdgesFromSourceTimer );
+ return ObservableTimer.time( edges, loadEdgesFromSourceByTypeTimer );
}
@@ -349,12 +347,12 @@ public class GraphManagerImpl implements GraphManager {
public Observable<Edge> loadEdgesToTargetByType( final SearchByIdType search ) {
final Observable<Edge> edges =
Observable.create( new ObservableIterator<MarkedEdge>( "getEdgeTypesFromSource" ) {
- @Override
- protected Iterator<MarkedEdge> getIterator() {
- return storageEdgeSerialization.getEdgesToTargetBySourceType( scope, search );
- }
- } ).buffer( graphFig.getScanPageSize() )
- .flatMap( new EdgeBufferFilter( search.getMaxTimestamp(), search.filterMarked() ) );
+ @Override
+ protected Iterator<MarkedEdge> getIterator() {
+ return storageEdgeSerialization.getEdgesToTargetBySourceType( scope, search );
+ }
+ } ).buffer( graphFig.getScanPageSize() )
+ .compose( new EdgeBufferFilter( search.getMaxTimestamp(), search.filterMarked() ) );
return ObservableTimer.time( edges, loadEdgesToTargetByTypeTimer );
}
@@ -362,13 +360,13 @@ public class GraphManagerImpl implements GraphManager {
@Override
public Observable<String> getEdgeTypesFromSource( final SearchEdgeType search ) {
- final Observable<String> edgeTypes = Observable.create(
- new ObservableIterator<String>( "getEdgeTypesFromSource" ) {
- @Override
- protected Iterator<String> getIterator() {
- return edgeMetadataSerialization.getEdgeTypesFromSource( scope, search );
- }
- } );
+ final Observable<String> edgeTypes =
+ Observable.create( new ObservableIterator<String>( "getEdgeTypesFromSource" ) {
+ @Override
+ protected Iterator<String> getIterator() {
+ return edgeMetadataSerialization.getEdgeTypesFromSource( scope, search );
+ }
+ } );
return ObservableTimer.time( edgeTypes, getEdgeTypesFromSourceTimer );
}
@@ -376,12 +374,13 @@ public class GraphManagerImpl implements GraphManager {
@Override
public Observable<String> getIdTypesFromSource( final SearchIdType search ) {
- final Observable<String> edgeTypes = Observable.create( new ObservableIterator<String>( "getIdTypesFromSource" ) {
- @Override
- protected Iterator<String> getIterator() {
- return edgeMetadataSerialization.getIdTypesFromSource( scope, search );
- }
- } );
+ final Observable<String> edgeTypes =
+ Observable.create( new ObservableIterator<String>( "getIdTypesFromSource" ) {
+ @Override
+ protected Iterator<String> getIterator() {
+ return edgeMetadataSerialization.getIdTypesFromSource( scope, search );
+ }
+ } );
return ObservableTimer.time( edgeTypes, getIdTypesFromSourceTimer );
}
@@ -389,22 +388,21 @@ public class GraphManagerImpl implements GraphManager {
@Override
public Observable<String> getEdgeTypesToTarget( final SearchEdgeType search ) {
- final Observable<String> edgeTypes = Observable.create(
- new ObservableIterator<String>( "getEdgeTypesToTarget" ) {
- @Override
- protected Iterator<String> getIterator() {
- return edgeMetadataSerialization.getEdgeTypesToTarget( scope, search );
- }
- } );
+ final Observable<String> edgeTypes =
+ Observable.create( new ObservableIterator<String>( "getEdgeTypesToTarget" ) {
+ @Override
+ protected Iterator<String> getIterator() {
+ return edgeMetadataSerialization.getEdgeTypesToTarget( scope, search );
+ }
+ } );
- return ObservableTimer.time( edgeTypes, getEdgeTypesFromSourceTimer );
+ return ObservableTimer.time( edgeTypes, getEdgeTypesToTargetTimer );
}
@Override
public Observable<String> getIdTypesToTarget( final SearchIdType search ) {
- final Observable<String> edgeTypes = Observable.create(
- new ObservableIterator<String>( "getIdTypesToTarget" ) {
+ final Observable<String> edgeTypes = Observable.create( new ObservableIterator<String>( "getIdTypesToTarget" ) {
@Override
protected Iterator<String> getIterator() {
return edgeMetadataSerialization.getIdTypesToTarget( scope, search );
@@ -418,7 +416,9 @@ public class GraphManagerImpl implements GraphManager {
/**
* Helper filter to perform mapping and return an observable of pre-filtered edges
*/
- private class EdgeBufferFilter implements Func1<List<MarkedEdge>, Observable<MarkedEdge>> {
+ private class EdgeBufferFilter implements
+ Observable.Transformer<List<MarkedEdge>, MarkedEdge> {//implements Func1<List<MarkedEdge>,
+ // Observable<MarkedEdge>> {
private final long maxVersion;
private final boolean filterMarked;
@@ -430,46 +430,60 @@ public class GraphManagerImpl implements GraphManager {
}
+ @Override
+
/**
* Takes a buffered list of marked edges. It then does a single round trip to fetch marked ids These are then
* used in conjunction with the max version filter to filter any edges that should not be returned
*
* @return An observable that emits only edges that can be consumed. There could be multiple versions of the
* same edge so those need de-duped.
- */
- @Override
- public Observable<MarkedEdge> call( final List<MarkedEdge> markedEdges ) {
+ */ public Observable<MarkedEdge> call( final Observable<List<MarkedEdge>> markedEdgesObservable ) {
- final Map<Id, Long> markedVersions = nodeSerialization.getMaxVersions( scope, markedEdges );
- final long maxTimestamp = maxVersion;
+ return markedEdgesObservable.flatMap( markedEdges -> {
- return Observable.from( markedEdges ).filter( edge -> {
- final long edgeTimestamp = edge.getTimestamp();
+ final Observable<MarkedEdge> markedEdgeObservable = Observable.from( markedEdges );
- //our edge needs to not be deleted and have a version that's > max Version
- if (( edge.isDeleted() && filterMarked) || Long.compare( edgeTimestamp, maxTimestamp ) > 0 ) {
- return false;
+ /**
+ * We aren't going to filter anything, return exactly what we're passed
+ */
+ if(!filterMarked){
+ return markedEdgeObservable;
}
+ //We need to filter, perform that filter
+ final Map<Id, Long> markedVersions = nodeSerialization.getMaxVersions( scope, markedEdges );
- final Long sourceTimestamp = markedVersions.get( edge.getSourceNode() );
+ return markedEdgeObservable.filter( edge -> {
+ final long edgeTimestamp = edge.getTimestamp();
- //the source Id has been marked for deletion. It's version is <= to the marked version for deletion,
- // so we need to discard it
- if ( sourceTimestamp != null && Long.compare( edgeTimestamp, sourceTimestamp ) < 1 ) {
- return false;
- }
+ //our edge needs to not be deleted and have a version that's > max Version
+ if ( edge.isDeleted() || Long.compare( edgeTimestamp, maxVersion ) > 0 ) {
+ return false;
+ }
- final Long targetTimestamp = markedVersions.get( edge.getTargetNode() );
- //the target Id has been marked for deletion. It's version is <= to the marked version for deletion,
- // so we need to discard it
- if ( targetTimestamp != null && Long.compare( edgeTimestamp, targetTimestamp ) < 1 ) {
- return false;
- }
+ final Long sourceTimestamp = markedVersions.get( edge.getSourceNode() );
+
+ //the source Id has been marked for deletion. It's version is <= to the marked version for
+ // deletion,
+ // so we need to discard it
+ if ( sourceTimestamp != null && Long.compare( edgeTimestamp, sourceTimestamp ) < 1 ) {
+ return false;
+ }
+
+ final Long targetTimestamp = markedVersions.get( edge.getTargetNode() );
+
+ //the target Id has been marked for deletion. It's version is <= to the marked version for
+ // deletion,
+ // so we need to discard it
+ if ( targetTimestamp != null && Long.compare( edgeTimestamp, targetTimestamp ) < 1 ) {
+ return false;
+ }
- return true;
+ return true;
+ } );
} );
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/62e650bc/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleSearchByIdType.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleSearchByIdType.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleSearchByIdType.java
index 9d989a3..bcbff0a 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleSearchByIdType.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleSearchByIdType.java
@@ -54,6 +54,14 @@ public class SimpleSearchByIdType extends SimpleSearchByEdgeType implements Sear
}
+ public SimpleSearchByIdType( final Id node, final String type, final long maxTimestamp, final Order order, final String idType,
+ final Optional<Edge> last, final boolean filterMarked ) {
+ super( node, type, maxTimestamp, order, last, filterMarked );
+ ValidationUtils.verifyString( idType, "idType" );
+ this.idType = idType;
+ }
+
+
@Override
public String getIdType() {
return idType;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/62e650bc/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerIT.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerIT.java
index 75f61f0..e512608 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerIT.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerIT.java
@@ -43,18 +43,21 @@ import com.google.inject.Inject;
import rx.Observable;
+import static org.apache.usergrid.persistence.core.util.IdGenerator.createId;
import static org.apache.usergrid.persistence.graph.test.util.EdgeTestUtils.createEdge;
import static org.apache.usergrid.persistence.graph.test.util.EdgeTestUtils.createGetByEdge;
-import static org.apache.usergrid.persistence.core.util.IdGenerator.createId;
import static org.apache.usergrid.persistence.graph.test.util.EdgeTestUtils.createSearchByEdge;
import static org.apache.usergrid.persistence.graph.test.util.EdgeTestUtils.createSearchByEdgeAndId;
+import static org.apache.usergrid.persistence.graph.test.util.EdgeTestUtils.createSearchByEdgeAndIdUnfiltered;
+import static org.apache.usergrid.persistence.graph.test.util.EdgeTestUtils.createSearchByEdgeUnfiltered;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
-@RunWith(ITRunner.class)
-@UseModules({ TestGraphModule.class })
+
+@RunWith( ITRunner.class )
+@UseModules( { TestGraphModule.class } )
public class GraphManagerIT {
@@ -69,18 +72,16 @@ public class GraphManagerIT {
protected ApplicationScope scope;
-
-
@Before
public void mockApp() {
- this.scope = new ApplicationScopeImpl(createId("application") );
+ this.scope = new ApplicationScopeImpl( createId( "application" ) );
}
@Test
public void testWriteReadEdgeTypeSource() throws TimeoutException, InterruptedException {
- GraphManager gm = emf.createEdgeManager( scope );
+ GraphManager gm = emf.createEdgeManager( scope );
Edge edge = createEdge( "source", "test", "target" );
@@ -112,7 +113,7 @@ public class GraphManagerIT {
@Test
public void testWriteReadEdgeTypeTarget() throws TimeoutException, InterruptedException {
- GraphManager gm = emf.createEdgeManager( scope );
+ GraphManager gm = emf.createEdgeManager( scope );
Edge edge = createEdge( "source", "test", "target" );
@@ -144,7 +145,7 @@ public class GraphManagerIT {
@Test
public void testWriteReadEdgeTypeVersionSource() throws TimeoutException, InterruptedException {
- GraphManager gm = emf.createEdgeManager( scope );
+ GraphManager gm = emf.createEdgeManager( scope );
final long earlyVersion = 1000l;
@@ -178,7 +179,7 @@ public class GraphManagerIT {
@Test
public void testWriteReadEdgeTypeVersionTarget() throws TimeoutException, InterruptedException {
- GraphManager gm = emf.createEdgeManager( scope );
+ GraphManager gm = emf.createEdgeManager( scope );
final long earlyVersion = 10000l;
@@ -216,7 +217,7 @@ public class GraphManagerIT {
@Test
public void testWriteReadEdgeTypeVersionSourceDistinct() throws TimeoutException, InterruptedException {
- GraphManager gm = emf.createEdgeManager( scope );
+ GraphManager gm = emf.createEdgeManager( scope );
final long earlyVersion = 10000l;
@@ -241,7 +242,7 @@ public class GraphManagerIT {
//now test retrieving it, we should only get edge3, since it's the latest
SearchByEdgeType search =
- createSearchByEdge( edge1.getSourceNode(), edge1.getType(), edge3.getTimestamp(), null );
+ createSearchByEdge( edge1.getSourceNode(), edge1.getType(), edge3.getTimestamp(), null );
Observable<Edge> edges = gm.loadEdgesFromSource( search );
@@ -288,7 +289,7 @@ public class GraphManagerIT {
public void testWriteReadEdgeTypeVersionTargetDistinct() throws TimeoutException, InterruptedException {
- GraphManager gm = emf.createEdgeManager( scope );
+ GraphManager gm = emf.createEdgeManager( scope );
final long earlyVersion = 10000l;
@@ -314,7 +315,7 @@ public class GraphManagerIT {
//now test retrieving it, we should only get edge3, since it's the latest
SearchByEdgeType search =
- createSearchByEdge( edge1.getTargetNode(), edge1.getType(), edge3.getTimestamp(), null );
+ createSearchByEdge( edge1.getTargetNode(), edge1.getType(), edge3.getTimestamp(), null );
Observable<Edge> edges = gm.loadEdgesToTarget( search );
@@ -360,7 +361,7 @@ public class GraphManagerIT {
@Test
public void testWriteReadEdgeTypePagingSource() throws TimeoutException, InterruptedException {
- GraphManager gm = emf.createEdgeManager( scope );
+ GraphManager gm = emf.createEdgeManager( scope );
final Id sourceId = IdGenerator.createId( "source" );
@@ -380,7 +381,7 @@ public class GraphManagerIT {
//now test retrieving it
SearchByEdgeType search =
- createSearchByEdge( edge1.getSourceNode(), edge1.getType(), edge3.getTimestamp(), null );
+ createSearchByEdge( edge1.getSourceNode(), edge1.getType(), edge3.getTimestamp(), null );
Observable<Edge> edges = gm.loadEdgesFromSource( search );
@@ -415,7 +416,7 @@ public class GraphManagerIT {
public void testWriteReadEdgeTypePagingTarget() {
- GraphManager gm = emf.createEdgeManager( scope );
+ GraphManager gm = emf.createEdgeManager( scope );
final Id targetId = IdGenerator.createId( "target" );
@@ -436,7 +437,7 @@ public class GraphManagerIT {
//now test retrieving it
SearchByEdgeType search =
- createSearchByEdge( edge1.getTargetNode(), edge1.getType(), edge3.getTimestamp(), null );
+ createSearchByEdge( edge1.getTargetNode(), edge1.getType(), edge3.getTimestamp(), null );
Observable<Edge> edges = gm.loadEdgesToTarget( search );
@@ -470,7 +471,7 @@ public class GraphManagerIT {
@Test
public void testWriteReadEdgeTypeTargetTypeSource() {
- GraphManager gm = emf.createEdgeManager( scope );
+ GraphManager gm = emf.createEdgeManager( scope );
Edge edge = createEdge( "source", "test", "target" );
@@ -480,7 +481,7 @@ public class GraphManagerIT {
//now test retrieving it
SearchByIdType search = createSearchByEdgeAndId( edge.getSourceNode(), edge.getType(), edge.getTimestamp(),
- edge.getTargetNode().getType(), null );
+ edge.getTargetNode().getType(), null );
Observable<Edge> edges = gm.loadEdgesFromSourceByType( search );
@@ -492,7 +493,7 @@ public class GraphManagerIT {
//change edge type to be invalid, shouldn't get a result
search = createSearchByEdgeAndId( edge.getSourceNode(), edge.getType(), edge.getTimestamp(),
- edge.getTargetNode().getType() + "invalid", null );
+ edge.getTargetNode().getType() + "invalid", null );
edges = gm.loadEdgesFromSourceByType( search );
@@ -506,7 +507,7 @@ public class GraphManagerIT {
@Test
public void testWriteReadEdgeTypeTargetTypeTarget() {
- GraphManager gm = emf.createEdgeManager( scope );
+ GraphManager gm = emf.createEdgeManager( scope );
;
@@ -517,7 +518,7 @@ public class GraphManagerIT {
//now test retrieving it
SearchByIdType search = createSearchByEdgeAndId( edge.getTargetNode(), edge.getType(), edge.getTimestamp(),
- edge.getSourceNode().getType(), null );
+ edge.getSourceNode().getType(), null );
Observable<Edge> edges = gm.loadEdgesToTargetByType( search );
@@ -529,7 +530,7 @@ public class GraphManagerIT {
//change edge type to be invalid, shouldn't get a result
search = createSearchByEdgeAndId( edge.getTargetNode(), edge.getType(), edge.getTimestamp(),
- edge.getSourceNode().getType() + "invalid", null );
+ edge.getSourceNode().getType() + "invalid", null );
edges = gm.loadEdgesToTargetByType( search );
@@ -543,7 +544,7 @@ public class GraphManagerIT {
@Test
public void testWriteReadEdgeDeleteSource() {
- GraphManager gm = emf.createEdgeManager( scope );
+ GraphManager gm = emf.createEdgeManager( scope );
Edge edge = createEdge( "source", "test", "target" );
@@ -563,7 +564,7 @@ public class GraphManagerIT {
assertEquals( "Correct edge returned", edge, returned );
SearchByIdType searchById = createSearchByEdgeAndId( edge.getSourceNode(), edge.getType(), edge.getTimestamp(),
- edge.getTargetNode().getType(), null );
+ edge.getTargetNode().getType(), null );
edges = gm.loadEdgesFromSourceByType( searchById );
@@ -573,8 +574,7 @@ public class GraphManagerIT {
assertEquals( "Correct edge returned", edge, returned );
final SearchByEdge searchByEdge =
- createGetByEdge( edge.getSourceNode(), edge.getType(), edge.getTargetNode(), edge.getTimestamp(),
- null );
+ createGetByEdge( edge.getSourceNode(), edge.getType(), edge.getTargetNode(), edge.getTimestamp(), null );
returned = gm.loadEdgeVersions( searchByEdge ).toBlocking().single();
@@ -613,7 +613,7 @@ public class GraphManagerIT {
@Test
public void testWriteReadEdgeDeleteTarget() {
- GraphManager gm = emf.createEdgeManager( scope );
+ GraphManager gm = emf.createEdgeManager( scope );
Edge edge = createEdge( "source", "test", "target" );
@@ -633,7 +633,7 @@ public class GraphManagerIT {
assertEquals( "Correct edge returned", edge, returned );
SearchByIdType searchById = createSearchByEdgeAndId( edge.getTargetNode(), edge.getType(), edge.getTimestamp(),
- edge.getSourceNode().getType(), null );
+ edge.getSourceNode().getType(), null );
edges = gm.loadEdgesToTargetByType( searchById );
@@ -669,7 +669,7 @@ public class GraphManagerIT {
@Test
public void testWriteReadEdgeTypesSourceTypes() {
- final GraphManager gm = emf.createEdgeManager( scope );
+ final GraphManager gm = emf.createEdgeManager( scope );
Id sourceId = new SimpleId( "source" );
Id targetId1 = new SimpleId( "target" );
@@ -691,7 +691,7 @@ public class GraphManagerIT {
//get our 2 edge types
Observable<String> edges =
- gm.getEdgeTypesFromSource( new SimpleSearchEdgeType( testTargetEdge.getSourceNode(), null, null ) );
+ gm.getEdgeTypesFromSource( new SimpleSearchEdgeType( testTargetEdge.getSourceNode(), null, null ) );
Iterator<String> results = edges.toBlocking().getIterator();
@@ -718,8 +718,8 @@ public class GraphManagerIT {
assertFalse( "No results", results.hasNext() );
//now get types for test2
- edges = gm.getIdTypesFromSource(
- new SimpleSearchIdType( testTargetEdge.getSourceNode(), "test2", null, null ) );
+ edges =
+ gm.getIdTypesFromSource( new SimpleSearchIdType( testTargetEdge.getSourceNode(), "test2", null, null ) );
results = edges.toBlocking().getIterator();
@@ -732,7 +732,7 @@ public class GraphManagerIT {
@Test
public void testWriteReadEdgeTypesTargetTypes() {
- final GraphManager gm = emf.createEdgeManager( scope );
+ final GraphManager gm = emf.createEdgeManager( scope );
Id sourceId1 = new SimpleId( "source" );
Id sourceId2 = new SimpleId( "source2" );
@@ -797,7 +797,7 @@ public class GraphManagerIT {
@Test
public void testWriteReadEdgeTypesSourceTypesPaging() {
- final GraphManager gm = emf.createEdgeManager( scope );
+ final GraphManager gm = emf.createEdgeManager( scope );
Id sourceId1 = new SimpleId( "source" );
Id targetId1 = new SimpleId( "target" );
@@ -861,7 +861,7 @@ public class GraphManagerIT {
//now get the next page
edges = gm.getIdTypesFromSource(
- new SimpleSearchIdType( testTargetEdge.getSourceNode(), "test", null, targetId1.getType() ) );
+ new SimpleSearchIdType( testTargetEdge.getSourceNode(), "test", null, targetId1.getType() ) );
results = edges.toBlocking().getIterator();
@@ -875,7 +875,7 @@ public class GraphManagerIT {
@Test
public void testWriteReadEdgeTypesTargetTypesPaging() {
- final GraphManager gm = emf.createEdgeManager( scope );
+ final GraphManager gm = emf.createEdgeManager( scope );
Id sourceId1 = new SimpleId( "source" );
Id sourceId2 = new SimpleId( "source2" );
@@ -942,7 +942,7 @@ public class GraphManagerIT {
//now get the next page
edges = gm.getIdTypesToTarget(
- new SimpleSearchIdType( testTargetEdge.getTargetNode(), "test", null, sourceId1.getType() ) );
+ new SimpleSearchIdType( testTargetEdge.getTargetNode(), "test", null, sourceId1.getType() ) );
results = edges.toBlocking().getIterator();
@@ -956,7 +956,7 @@ public class GraphManagerIT {
@Test
public void testMarkSourceEdges() throws InterruptedException {
- final GraphManager gm = emf.createEdgeManager( scope );
+ final GraphManager gm = emf.createEdgeManager( scope );
Id sourceId = new SimpleId( "source" );
Id targetId1 = new SimpleId( "target" );
@@ -966,11 +966,11 @@ public class GraphManagerIT {
long startTime = System.currentTimeMillis();
long edge1Time = startTime;
- long edge2Time = edge1Time+1;
+ long edge2Time = edge1Time + 1;
- final long maxVersion= edge2Time;
+ final long maxVersion = edge2Time;
- Edge edge1 = createEdge( sourceId, "test", targetId1, edge1Time);
+ Edge edge1 = createEdge( sourceId, "test", targetId1, edge1Time );
gm.writeEdge( edge1 ).toBlocking().singleOrDefault( null );
@@ -979,21 +979,18 @@ public class GraphManagerIT {
gm.writeEdge( edge2 ).toBlocking().singleOrDefault( null );
-
assertTrue( Long.compare( maxVersion, edge2.getTimestamp() ) >= 0 );
assertTrue( Long.compare( maxVersion, edge1.getTimestamp() ) >= 0 );
//get our 2 edges
- Observable<Edge> edges = gm.loadEdgesFromSource(
- createSearchByEdge( edge1.getSourceNode(), edge1.getType(), maxVersion, null ) );
+ Observable<Edge> edges =
+ gm.loadEdgesFromSource( createSearchByEdge( edge1.getSourceNode(), edge1.getType(), maxVersion, null ) );
Iterator<Edge> results = edges.toBlocking().getIterator();
- System.out.println( "\n\n\n\n\n\n\n\n\n\n" );
-
assertEquals( "Edges correct", edge2, results.next() );
assertEquals( "Edges correct", edge1, results.next() );
@@ -1002,15 +999,12 @@ public class GraphManagerIT {
//now delete one of the edges
- System.out.println( "\n\n\n\n\n\n\n\n\n\n" );
gm.markEdge( edge1 ).toBlocking().last();
- System.out.println( "\n\n\n\n\n\n\n\n\n\n" );
-
- edges = gm.loadEdgesFromSource(
- createSearchByEdge( edge1.getSourceNode(), edge1.getType(), maxVersion, null ) );
+ edges =
+ gm.loadEdgesFromSource( createSearchByEdge( edge1.getSourceNode(), edge1.getType(), maxVersion, null ) );
results = edges.toBlocking().getIterator();
@@ -1020,16 +1014,14 @@ public class GraphManagerIT {
assertFalse( "No more edges", results.hasNext() );
- System.out.println( "\n\n\n\n\n\n\n\n\n\n" );
//now delete one of the edges
gm.markEdge( edge2 ).toBlocking().last();
- System.out.println( "\n\n\n\n\n\n\n\n\n\n" );
- edges = gm.loadEdgesFromSource(
- createSearchByEdge( edge1.getSourceNode(), edge1.getType(), maxVersion, null ) );
+ edges =
+ gm.loadEdgesFromSource( createSearchByEdge( edge1.getSourceNode(), edge1.getType(), maxVersion, null ) );
results = edges.toBlocking().getIterator();
@@ -1045,7 +1037,7 @@ public class GraphManagerIT {
@Test
public void testMarkTargetEdges() {
- final GraphManager gm = emf.createEdgeManager( scope );
+ final GraphManager gm = emf.createEdgeManager( scope );
Id sourceId1 = new SimpleId( "source" );
Id sourceId2 = new SimpleId( "source2" );
@@ -1065,7 +1057,7 @@ public class GraphManagerIT {
//get our 2 edges
Observable<Edge> edges =
- gm.loadEdgesToTarget( createSearchByEdge( edge1.getTargetNode(), edge1.getType(), maxVersion, null ) );
+ gm.loadEdgesToTarget( createSearchByEdge( edge1.getTargetNode(), edge1.getType(), maxVersion, null ) );
Iterator<Edge> results = edges.toBlocking().getIterator();
@@ -1111,9 +1103,163 @@ public class GraphManagerIT {
@Test
+ public void testMarkCompactSourceEdges() throws InterruptedException {
+
+ final GraphManager gm = emf.createEdgeManager( scope );
+
+ Id sourceId = new SimpleId( "source" );
+ Id targetId1 = new SimpleId( "target" );
+ Id targetId2 = new SimpleId( "target2" );
+
+
+ long startTime = System.currentTimeMillis();
+
+ long edge1Time = startTime;
+ long edge2Time = edge1Time + 1;
+
+ final long maxVersion = edge2Time;
+
+ Edge edge1 = createEdge( sourceId, "test", targetId1, edge1Time );
+
+ gm.writeEdge( edge1 ).toBlocking().singleOrDefault( null );
+
+ Edge edge2 = createEdge( sourceId, "test", targetId2, edge2Time );
+
+ gm.writeEdge( edge2 ).toBlocking().singleOrDefault( null );
+
+
+ assertTrue( Long.compare( maxVersion, edge2.getTimestamp() ) >= 0 );
+ assertTrue( Long.compare( maxVersion, edge1.getTimestamp() ) >= 0 );
+
+
+ gm.markEdge( edge1 ).toBlocking().last();
+
+
+ //get our 2 edges
+ Observable<Edge> edges = gm.loadEdgesFromSource(
+ createSearchByEdgeUnfiltered( edge1.getSourceNode(), edge1.getType(), maxVersion, null ) );
+
+
+ Iterator<Edge> results = edges.toBlocking().getIterator();
+
+
+ assertEquals( "Edges correct", edge2.getTargetNode(), results.next().getTargetNode() );
+
+ assertEquals( "Edges correct", edge1.getTargetNode(), results.next().getTargetNode() );
+
+ assertFalse( "No more edges", results.hasNext() );
+
+
+ //now delete one of the edges
+
+ gm.deleteEdge( edge1 ).toBlocking().last();
+
+
+ edges = gm.loadEdgesFromSource(
+ createSearchByEdgeUnfiltered( edge1.getSourceNode(), edge1.getType(), maxVersion, null ) );
+
+
+ results = edges.toBlocking().getIterator();
+
+ assertEquals( "Edges correct", edge2.getTargetNode(), results.next().getTargetNode() );
+
+ assertFalse( "No more edges", results.hasNext() );
+
+ //delete an edge we didn't mark, should be a no-op
+ gm.deleteEdge( edge2 ).toBlocking().lastOrDefault( null );
+
+ edges = gm.loadEdgesFromSource(
+ createSearchByEdgeUnfiltered( edge1.getSourceNode(), edge1.getType(), maxVersion, null ) );
+
+
+ results = edges.toBlocking().getIterator();
+
+
+ assertEquals( "Edges correct", edge2.getSourceNode(), results.next().getSourceNode() );
+
+ assertFalse( "No more edges", results.hasNext() );
+ }
+
+
+ @Test
+ public void testMarkCompactTargetEdges() {
+
+ final GraphManager gm = emf.createEdgeManager( scope );
+
+ Id sourceId1 = new SimpleId( "source" );
+ Id sourceId2 = new SimpleId( "source2" );
+ Id targetId = new SimpleId( "target" );
+
+ Edge edge1 = createEdge( sourceId1, "test", targetId, System.currentTimeMillis() );
+
+ gm.writeEdge( edge1 ).toBlocking().last();
+
+ Edge edge2 = createEdge( sourceId2, "test", targetId, System.currentTimeMillis() );
+
+ gm.writeEdge( edge2 ).toBlocking().last();
+
+
+ final long maxVersion = System.currentTimeMillis();
+
+
+ //now delete one of the edges
+
+ gm.markEdge( edge1 ).toBlocking().last();
+
+ //get our 2 edges
+ Observable<Edge> edges = gm.loadEdgesToTarget(
+ createSearchByEdgeUnfiltered( edge1.getTargetNode(), edge1.getType(), maxVersion, null ) );
+
+
+ Iterator<Edge> results = edges.toBlocking().getIterator();
+
+
+ assertEquals( "Edges correct", edge2.getSourceNode(), results.next().getSourceNode() );
+
+ assertEquals( "Edges correct", edge1.getSourceNode(), results.next().getSourceNode() );
+
+
+ assertFalse( "No more edges", results.hasNext() );
+
+
+ //now delete one of the edges
+
+ gm.deleteEdge( edge1 ).toBlocking().last();
+
+ edges = gm.loadEdgesToTarget(
+ createSearchByEdgeUnfiltered( edge1.getTargetNode(), edge1.getType(), maxVersion, null ) );
+
+
+ results = edges.toBlocking().getIterator();
+
+
+ assertEquals( "Edges correct", edge2.getSourceNode(), results.next().getSourceNode() );
+
+ assertFalse( "No more edges", results.hasNext() );
+
+ //now delete one of the edges
+
+
+ //delete an edge we didn't mark, should be a no-op
+ gm.deleteEdge( edge2 ).toBlocking().lastOrDefault( null );
+
+ edges = gm.loadEdgesToTarget(
+ createSearchByEdgeUnfiltered( edge1.getTargetNode(), edge1.getType(), maxVersion, null ) );
+
+
+ results = edges.toBlocking().getIterator();
+
+
+ assertEquals( "Edges correct", edge2.getSourceNode(), results.next().getSourceNode() );
+
+ assertFalse( "No more edges", results.hasNext() );
+ }
+
+
+ @Test
public void testMarkSourceEdgesType() {
- final GraphManager gm = emf.createEdgeManager( scope );
+ final GraphManager gm = emf.createEdgeManager( scope );
Id sourceId = new SimpleId( "source" );
Id targetId1 = new SimpleId( "target" );
@@ -1133,7 +1279,7 @@ public class GraphManagerIT {
//get our 2 edges
Observable<Edge> edges = gm.loadEdgesFromSourceByType(
- createSearchByEdgeAndId( sourceId, edge1.getType(), maxVersion, targetId1.getType(), null ) );
+ createSearchByEdgeAndId( sourceId, edge1.getType(), maxVersion, targetId1.getType(), null ) );
Iterator<Edge> results = edges.toBlocking().getIterator();
@@ -1149,7 +1295,7 @@ public class GraphManagerIT {
edges = gm.loadEdgesFromSourceByType(
- createSearchByEdgeAndId( sourceId, edge1.getType(), maxVersion, targetId1.getType(), null ) );
+ createSearchByEdgeAndId( sourceId, edge1.getType(), maxVersion, targetId1.getType(), null ) );
results = edges.toBlocking().getIterator();
@@ -1158,7 +1304,7 @@ public class GraphManagerIT {
edges = gm.loadEdgesFromSourceByType(
- createSearchByEdgeAndId( sourceId, edge1.getType(), maxVersion, targetId2.getType(), null ) );
+ createSearchByEdgeAndId( sourceId, edge1.getType(), maxVersion, targetId2.getType(), null ) );
results = edges.toBlocking().getIterator();
@@ -1174,7 +1320,7 @@ public class GraphManagerIT {
edges = gm.loadEdgesFromSourceByType(
- createSearchByEdgeAndId( sourceId, edge1.getType(), maxVersion, targetId2.getType(), null ) );
+ createSearchByEdgeAndId( sourceId, edge1.getType(), maxVersion, targetId2.getType(), null ) );
results = edges.toBlocking().getIterator();
@@ -1191,7 +1337,7 @@ public class GraphManagerIT {
@Test
public void testMarkTargetEdgesType() {
- final GraphManager gm = emf.createEdgeManager( scope );
+ final GraphManager gm = emf.createEdgeManager( scope );
Id sourceId1 = new SimpleId( "source" );
Id sourceId2 = new SimpleId( "source2" );
@@ -1210,7 +1356,7 @@ public class GraphManagerIT {
//get our 2 edges
Observable<Edge> edges = gm.loadEdgesToTargetByType(
- createSearchByEdgeAndId( targetId, edge1.getType(), maxVersion, sourceId1.getType(), null ) );
+ createSearchByEdgeAndId( targetId, edge1.getType(), maxVersion, sourceId1.getType(), null ) );
Iterator<Edge> results = edges.toBlocking().getIterator();
@@ -1226,8 +1372,7 @@ public class GraphManagerIT {
edges = gm.loadEdgesToTargetByType(
- createSearchByEdgeAndId( edge1.getSourceNode(), edge1.getType(), maxVersion, sourceId1.getType(),
- null ) );
+ createSearchByEdgeAndId( edge1.getSourceNode(), edge1.getType(), maxVersion, sourceId1.getType(), null ) );
results = edges.toBlocking().getIterator();
@@ -1236,7 +1381,7 @@ public class GraphManagerIT {
edges = gm.loadEdgesToTargetByType(
- createSearchByEdgeAndId( targetId, edge1.getType(), maxVersion, sourceId2.getType(), null ) );
+ createSearchByEdgeAndId( targetId, edge1.getType(), maxVersion, sourceId2.getType(), null ) );
results = edges.toBlocking().getIterator();
@@ -1252,7 +1397,7 @@ public class GraphManagerIT {
edges = gm.loadEdgesToTargetByType(
- createSearchByEdgeAndId( targetId, edge1.getType(), maxVersion, sourceId2.getType(), null ) );
+ createSearchByEdgeAndId( targetId, edge1.getType(), maxVersion, sourceId2.getType(), null ) );
results = edges.toBlocking().getIterator();
@@ -1269,7 +1414,7 @@ public class GraphManagerIT {
@Test
public void markSourceNode() {
- final GraphManager gm = emf.createEdgeManager( scope );
+ final GraphManager gm = emf.createEdgeManager( scope );
Id sourceId = new SimpleId( "source" );
Id targetId1 = new SimpleId( "target" );
@@ -1287,8 +1432,8 @@ public class GraphManagerIT {
final long maxVersion = System.currentTimeMillis();
Iterator<Edge> results =
- gm.loadEdgesFromSource( createSearchByEdge( sourceId, edge1.getType(), maxVersion, null ) )
- .toBlocking().getIterator();
+ gm.loadEdgesFromSource( createSearchByEdge( sourceId, edge1.getType(), maxVersion, null ) ).toBlocking()
+ .getIterator();
assertEquals( "Edge found", edge2, results.next() );
@@ -1300,8 +1445,8 @@ public class GraphManagerIT {
//get our 2 edges
results = gm.loadEdgesFromSourceByType(
- createSearchByEdgeAndId( sourceId, edge1.getType(), maxVersion, targetId1.getType(), null ) )
- .toBlocking().getIterator();
+ createSearchByEdgeAndId( sourceId, edge1.getType(), maxVersion, targetId1.getType(), null ) ).toBlocking()
+ .getIterator();
assertEquals( "Edges correct", edge1, results.next() );
@@ -1310,8 +1455,8 @@ public class GraphManagerIT {
//now delete one of the edges
results = gm.loadEdgesFromSourceByType(
- createSearchByEdgeAndId( sourceId, edge2.getType(), maxVersion, targetId2.getType(), null ) )
- .toBlocking().getIterator();
+ createSearchByEdgeAndId( sourceId, edge2.getType(), maxVersion, targetId2.getType(), null ) ).toBlocking()
+ .getIterator();
assertEquals( "Edges correct", edge2, results.next() );
@@ -1324,26 +1469,61 @@ public class GraphManagerIT {
//now re-read, nothing should be there since they're marked
- results = gm.loadEdgesFromSource( createSearchByEdge( sourceId, edge1.getType(), maxVersion, null ) )
- .toBlocking().getIterator();
+ results =
+ gm.loadEdgesFromSource( createSearchByEdge( sourceId, edge1.getType(), maxVersion, null ) ).toBlocking()
+ .getIterator();
assertFalse( "No more edges", results.hasNext() );
//get our 2 edges
results = gm.loadEdgesFromSourceByType(
- createSearchByEdgeAndId( sourceId, edge1.getType(), maxVersion, targetId1.getType(), null ) )
- .toBlocking().getIterator();
+ createSearchByEdgeAndId( sourceId, edge1.getType(), maxVersion, targetId1.getType(), null ) ).toBlocking()
+ .getIterator();
assertFalse( "No more edges", results.hasNext() );
+
//now delete one of the edges
results = gm.loadEdgesFromSourceByType(
- createSearchByEdgeAndId( sourceId, edge2.getType(), maxVersion, targetId2.getType(), null ) )
+ createSearchByEdgeAndId( sourceId, edge2.getType(), maxVersion, targetId2.getType(), null ) ).toBlocking()
+ .getIterator();
+
+
+ assertFalse( "No more edges", results.hasNext() );
+
+
+ //test they come back unfiltered
+
+ //read with filter marked off to ensure they're still persisted
+ results = gm.loadEdgesFromSource( createSearchByEdgeUnfiltered( sourceId, edge1.getType(), maxVersion, null ) )
+ .toBlocking().getIterator();
+
+
+ assertEquals( "Edge found", edge2, results.next() );
+
+ assertEquals( "Edge found", edge1, results.next() );
+
+ assertFalse( "No more edges", results.hasNext() );
+
+
+ results = gm.loadEdgesFromSourceByType(
+ createSearchByEdgeAndIdUnfiltered( sourceId, edge1.getType(), maxVersion, targetId1.getType(), null ) )
.toBlocking().getIterator();
+ assertEquals( "Edges correct", edge1, results.next() );
+
+ assertFalse( "No more edges", results.hasNext() );
+
+ results = gm.loadEdgesFromSourceByType(
+ createSearchByEdgeAndIdUnfiltered( sourceId, edge2.getType(), maxVersion, targetId2.getType(), null ) )
+ .toBlocking().getIterator();
+
+
+ assertEquals( "Edges correct", edge2, results.next() );
+
assertFalse( "No more edges", results.hasNext() );
}
@@ -1351,7 +1531,7 @@ public class GraphManagerIT {
@Test
public void markTargetNode() {
- final GraphManager gm = emf.createEdgeManager( scope );
+ final GraphManager gm = emf.createEdgeManager( scope );
Id sourceId1 = new SimpleId( "source" );
Id sourceId2 = new SimpleId( "source2" );
@@ -1369,8 +1549,8 @@ public class GraphManagerIT {
final long maxVersion = System.currentTimeMillis();
Iterator<Edge> results =
- gm.loadEdgesToTarget( createSearchByEdge( targetId, edge1.getType(), maxVersion, null ) )
- .toBlocking().getIterator();
+ gm.loadEdgesToTarget( createSearchByEdge( targetId, edge1.getType(), maxVersion, null ) ).toBlocking()
+ .getIterator();
assertEquals( "Edge found", edge2, results.next() );
@@ -1382,7 +1562,69 @@ public class GraphManagerIT {
//get our 2 edges
results = gm.loadEdgesToTargetByType(
- createSearchByEdgeAndId( targetId, edge1.getType(), maxVersion, sourceId1.getType(), null ) )
+ createSearchByEdgeAndId( targetId, edge1.getType(), maxVersion, sourceId1.getType(), null ) ).toBlocking()
+ .getIterator();
+
+
+ assertEquals( "Edges correct", edge1, results.next() );
+
+ assertFalse( "No more edges", results.hasNext() );
+
+ //now delete one of the edges
+ results = gm.loadEdgesToTargetByType(
+ createSearchByEdgeAndId( targetId, edge2.getType(), maxVersion, sourceId2.getType(), null ) ).toBlocking()
+ .getIterator();
+
+
+ assertEquals( "Edges correct", edge2, results.next() );
+
+ assertFalse( "No more edges", results.hasNext() );
+
+ //mark the source node
+ gm.markNode( targetId, edge2.getTimestamp() ).toBlocking().last();
+
+
+ //now re-read, nothing should be there since they're marked
+
+ results = gm.loadEdgesToTarget( createSearchByEdge( targetId, edge1.getType(), maxVersion, null ) ).toBlocking()
+ .getIterator();
+
+ assertFalse( "No more edges", results.hasNext() );
+
+
+ //get our 2 edges
+ results = gm.loadEdgesToTargetByType(
+ createSearchByEdgeAndId( targetId, edge1.getType(), maxVersion, sourceId1.getType(), null ) ).toBlocking()
+ .getIterator();
+
+
+ assertFalse( "No more edges", results.hasNext() );
+
+ //now delete one of the edges
+ results = gm.loadEdgesToTargetByType(
+ createSearchByEdgeAndId( targetId, edge2.getType(), maxVersion, sourceId2.getType(), null ) ).toBlocking()
+ .getIterator();
+
+
+ assertFalse( "No more edges", results.hasNext() );
+
+
+ //now test they come back when unfiltered
+
+ results = gm.loadEdgesToTarget( createSearchByEdgeUnfiltered( targetId, edge1.getType(), maxVersion, null ) )
+ .toBlocking().getIterator();
+
+
+ assertEquals( "Edge found", edge2, results.next() );
+
+ assertEquals( "Edge found", edge1, results.next() );
+
+ assertFalse( "No more edges", results.hasNext() );
+
+
+ //get our 2 edges
+ results = gm.loadEdgesToTargetByType(
+ createSearchByEdgeAndIdUnfiltered( targetId, edge1.getType(), maxVersion, sourceId1.getType(), null ) )
.toBlocking().getIterator();
@@ -1392,10 +1634,180 @@ public class GraphManagerIT {
//now delete one of the edges
results = gm.loadEdgesToTargetByType(
- createSearchByEdgeAndId( targetId, edge2.getType(), maxVersion, sourceId2.getType(), null ) )
+ createSearchByEdgeAndIdUnfiltered( targetId, edge2.getType(), maxVersion, sourceId2.getType(), null ) )
+ .toBlocking().getIterator();
+
+
+ assertEquals( "Edges correct", edge2, results.next() );
+
+ assertFalse( "No more edges", results.hasNext() );
+ }
+
+
+ @Test
+ public void markDeleteSourceNode() {
+
+ final GraphManager gm = emf.createEdgeManager( scope );
+
+ Id sourceId = new SimpleId( "source" );
+ Id targetId1 = new SimpleId( "target" );
+ Id targetId2 = new SimpleId( "target2" );
+
+ Edge edge1 = createEdge( sourceId, "test", targetId1, System.currentTimeMillis() );
+
+ gm.writeEdge( edge1 ).toBlocking().singleOrDefault( null );
+
+ Edge edge2 = createEdge( sourceId, "test", targetId2, System.currentTimeMillis() );
+
+ gm.writeEdge( edge2 ).toBlocking().singleOrDefault( null );
+
+
+ final long maxVersion = System.currentTimeMillis();
+
+ Iterator<Edge> results =
+ gm.loadEdgesFromSource( createSearchByEdge( sourceId, edge1.getType(), maxVersion, null ) ).toBlocking()
+ .getIterator();
+
+
+ assertEquals( "Edge found", edge2, results.next() );
+
+ assertEquals( "Edge found", edge1, results.next() );
+
+ assertFalse( "No more edges", results.hasNext() );
+
+
+ //get our 2 edges
+ results = gm.loadEdgesFromSourceByType(
+ createSearchByEdgeAndId( sourceId, edge1.getType(), maxVersion, targetId1.getType(), null ) ).toBlocking()
+ .getIterator();
+
+
+ assertEquals( "Edges correct", edge1, results.next() );
+
+ assertFalse( "No more edges", results.hasNext() );
+
+ //now delete one of the edges
+ results = gm.loadEdgesFromSourceByType(
+ createSearchByEdgeAndId( sourceId, edge2.getType(), maxVersion, targetId2.getType(), null ) ).toBlocking()
+ .getIterator();
+
+
+ assertEquals( "Edges correct", edge2, results.next() );
+
+ assertFalse( "No more edges", results.hasNext() );
+
+ //mark the source node
+ gm.markNode( sourceId, edge2.getTimestamp() ).toBlocking().last();
+
+
+ //test they come back unfiltered
+
+ //read with filter marked off to ensure they're still persisted
+ results = gm.loadEdgesFromSource( createSearchByEdgeUnfiltered( sourceId, edge1.getType(), maxVersion, null ) )
+ .toBlocking().getIterator();
+
+
+ assertEquals( "Edge found", edge2, results.next() );
+
+ assertEquals( "Edge found", edge1, results.next() );
+
+ assertFalse( "No more edges", results.hasNext() );
+
+
+ results = gm.loadEdgesFromSourceByType(
+ createSearchByEdgeAndIdUnfiltered( sourceId, edge1.getType(), maxVersion, targetId1.getType(), null ) )
+ .toBlocking().getIterator();
+
+
+ assertEquals( "Edges correct", edge1, results.next() );
+
+ assertFalse( "No more edges", results.hasNext() );
+
+ results = gm.loadEdgesFromSourceByType(
+ createSearchByEdgeAndIdUnfiltered( sourceId, edge2.getType(), maxVersion, targetId2.getType(), null ) )
+ .toBlocking().getIterator();
+
+
+ assertEquals( "Edges correct", edge2, results.next() );
+
+ assertFalse( "No more edges", results.hasNext() );
+
+ //now delete them
+
+
+ gm.compactNode( sourceId ).toBlocking().last();
+
+ results = gm.loadEdgesFromSource( createSearchByEdgeUnfiltered( sourceId, edge1.getType(), maxVersion, null ) )
.toBlocking().getIterator();
+ assertFalse( "No more edges", results.hasNext() );
+
+
+ results = gm.loadEdgesFromSourceByType(
+ createSearchByEdgeAndIdUnfiltered( sourceId, edge1.getType(), maxVersion, targetId1.getType(), null ) )
+ .toBlocking().getIterator();
+
+
+ assertFalse( "No more edges", results.hasNext() );
+
+ results = gm.loadEdgesFromSourceByType(
+ createSearchByEdgeAndIdUnfiltered( sourceId, edge2.getType(), maxVersion, targetId2.getType(), null ) )
+ .toBlocking().getIterator();
+
+
+ assertFalse( "No more edges", results.hasNext() );
+ }
+
+
+ @Test
+ public void markDeleteTargetNode() {
+
+ final GraphManager gm = emf.createEdgeManager( scope );
+
+ Id sourceId1 = new SimpleId( "source" );
+ Id sourceId2 = new SimpleId( "source2" );
+ Id targetId = new SimpleId( "target" );
+
+ Edge edge1 = createEdge( sourceId1, "test", targetId, System.currentTimeMillis() );
+
+ gm.writeEdge( edge1 ).toBlocking().singleOrDefault( null );
+
+ Edge edge2 = createEdge( sourceId2, "test", targetId, System.currentTimeMillis() );
+
+ gm.writeEdge( edge2 ).toBlocking().singleOrDefault( null );
+
+
+ final long maxVersion = System.currentTimeMillis();
+
+ Iterator<Edge> results =
+ gm.loadEdgesToTarget( createSearchByEdge( targetId, edge1.getType(), maxVersion, null ) ).toBlocking()
+ .getIterator();
+
+
+ assertEquals( "Edge found", edge2, results.next() );
+
+ assertEquals( "Edge found", edge1, results.next() );
+
+ assertFalse( "No more edges", results.hasNext() );
+
+
+ //get our 2 edges
+ results = gm.loadEdgesToTargetByType(
+ createSearchByEdgeAndId( targetId, edge1.getType(), maxVersion, sourceId1.getType(), null ) ).toBlocking()
+ .getIterator();
+
+
+ assertEquals( "Edges correct", edge1, results.next() );
+
+ assertFalse( "No more edges", results.hasNext() );
+
+ //now delete one of the edges
+ results = gm.loadEdgesToTargetByType(
+ createSearchByEdgeAndId( targetId, edge2.getType(), maxVersion, sourceId2.getType(), null ) ).toBlocking()
+ .getIterator();
+
+
assertEquals( "Edges correct", edge2, results.next() );
assertFalse( "No more edges", results.hasNext() );
@@ -1406,15 +1818,55 @@ public class GraphManagerIT {
//now re-read, nothing should be there since they're marked
- results = gm.loadEdgesToTarget( createSearchByEdge( targetId, edge1.getType(), maxVersion, null ) )
+
+ //now test they come back when unfiltered
+
+ results = gm.loadEdgesToTarget( createSearchByEdgeUnfiltered( targetId, edge1.getType(), maxVersion, null ) )
+ .toBlocking().getIterator();
+
+
+ assertEquals( "Edge found", edge2, results.next() );
+
+ assertEquals( "Edge found", edge1, results.next() );
+
+ assertFalse( "No more edges", results.hasNext() );
+
+
+ //get our 2 edges
+ results = gm.loadEdgesToTargetByType(
+ createSearchByEdgeAndIdUnfiltered( targetId, edge1.getType(), maxVersion, sourceId1.getType(), null ) )
+ .toBlocking().getIterator();
+
+
+ assertEquals( "Edges correct", edge1, results.next() );
+
+ assertFalse( "No more edges", results.hasNext() );
+
+ //now delete one of the edges
+ results = gm.loadEdgesToTargetByType(
+ createSearchByEdgeAndIdUnfiltered( targetId, edge2.getType(), maxVersion, sourceId2.getType(), null ) )
.toBlocking().getIterator();
+
+ assertEquals( "Edges correct", edge2, results.next() );
+
+ assertFalse( "No more edges", results.hasNext() );
+
+ //now compact, everything should be removed
+
+ gm.compactNode( targetId ).toBlocking().last();
+
+
+ results = gm.loadEdgesToTarget( createSearchByEdgeUnfiltered( targetId, edge1.getType(), maxVersion, null ) )
+ .toBlocking().getIterator();
+
+
assertFalse( "No more edges", results.hasNext() );
//get our 2 edges
results = gm.loadEdgesToTargetByType(
- createSearchByEdgeAndId( targetId, edge1.getType(), maxVersion, sourceId1.getType(), null ) )
+ createSearchByEdgeAndIdUnfiltered( targetId, edge1.getType(), maxVersion, sourceId1.getType(), null ) )
.toBlocking().getIterator();
@@ -1422,7 +1874,7 @@ public class GraphManagerIT {
//now delete one of the edges
results = gm.loadEdgesToTargetByType(
- createSearchByEdgeAndId( targetId, edge2.getType(), maxVersion, sourceId2.getType(), null ) )
+ createSearchByEdgeAndIdUnfiltered( targetId, edge2.getType(), maxVersion, sourceId2.getType(), null ) )
.toBlocking().getIterator();
@@ -1433,7 +1885,7 @@ public class GraphManagerIT {
@Test
public void testWriteReadEdgeTypesSourceTypesPrefix() {
- final GraphManager gm = emf.createEdgeManager( scope );
+ final GraphManager gm = emf.createEdgeManager( scope );
Id sourceId = new SimpleId( "source" );
Id targetId = new SimpleId( "target" );
@@ -1454,7 +1906,7 @@ public class GraphManagerIT {
//get our 2 edge types
Observable<String> edges =
- gm.getEdgeTypesFromSource( new SimpleSearchEdgeType( testTargetEdge.getSourceNode(), "test1", null ) );
+ gm.getEdgeTypesFromSource( new SimpleSearchEdgeType( testTargetEdge.getSourceNode(), "test1", null ) );
Iterator<String> results = edges.toBlocking().getIterator();
@@ -1483,7 +1935,7 @@ public class GraphManagerIT {
public void testSourceSubTypes() {
//now test sub edges
- final GraphManager gm = emf.createEdgeManager( scope );
+ final GraphManager gm = emf.createEdgeManager( scope );
Id sourceId = new SimpleId( "source" );
Id targetId1target1 = new SimpleId( "type1target1" );
@@ -1504,8 +1956,8 @@ public class GraphManagerIT {
gm.writeEdge( test2TargetEdge ).toBlocking().singleOrDefault( null );
- Observable<String> edges = gm.getIdTypesFromSource(
- new SimpleSearchIdType( testTargetEdge.getSourceNode(), "test", "type1", null ) );
+ Observable<String> edges =
+ gm.getIdTypesFromSource( new SimpleSearchIdType( testTargetEdge.getSourceNode(), "test", "type1", null ) );
Iterator<String> results = edges.toBlocking().getIterator();
@@ -1517,8 +1969,8 @@ public class GraphManagerIT {
assertFalse( "No results", results.hasNext() );
//now get types for test2
- edges = gm.getIdTypesFromSource(
- new SimpleSearchIdType( testTargetEdge.getSourceNode(), "test", "type2", null ) );
+ edges =
+ gm.getIdTypesFromSource( new SimpleSearchIdType( testTargetEdge.getSourceNode(), "test", "type2", null ) );
results = edges.toBlocking().getIterator();
@@ -1532,7 +1984,7 @@ public class GraphManagerIT {
@Test
public void testWriteReadEdgeTypesTargetTypesPrefix() {
- final GraphManager gm = emf.createEdgeManager( scope );
+ final GraphManager gm = emf.createEdgeManager( scope );
Id targetId = new SimpleId( "target" );
Id sourceId = new SimpleId( "source" );
@@ -1553,7 +2005,7 @@ public class GraphManagerIT {
//get our 2 edge types
Observable<String> edges =
- gm.getEdgeTypesToTarget( new SimpleSearchEdgeType( testTargetEdge.getTargetNode(), "test1", null ) );
+ gm.getEdgeTypesToTarget( new SimpleSearchEdgeType( testTargetEdge.getTargetNode(), "test1", null ) );
Iterator<String> results = edges.toBlocking().getIterator();
@@ -1582,7 +2034,7 @@ public class GraphManagerIT {
public void testTargetSubTypes() {
//now test sub edges
- final GraphManager gm = emf.createEdgeManager( scope );
+ final GraphManager gm = emf.createEdgeManager( scope );
Id targetId = new SimpleId( "target" );
Id sourceId1target1 = new SimpleId( "type1source1" );
@@ -1603,8 +2055,8 @@ public class GraphManagerIT {
gm.writeEdge( test2TargetEdge ).toBlocking().singleOrDefault( null );
- Observable<String> edges = gm.getIdTypesToTarget(
- new SimpleSearchIdType( testTargetEdge.getTargetNode(), "test", "type1", null ) );
+ Observable<String> edges =
+ gm.getIdTypesToTarget( new SimpleSearchIdType( testTargetEdge.getTargetNode(), "test", "type1", null ) );
Iterator<String> results = edges.toBlocking().getIterator();
@@ -1616,8 +2068,8 @@ public class GraphManagerIT {
assertFalse( "No results", results.hasNext() );
//now get types for test2
- edges = gm.getIdTypesToTarget(
- new SimpleSearchIdType( testTargetEdge.getTargetNode(), "test", "type2", null ) );
+ edges =
+ gm.getIdTypesToTarget( new SimpleSearchIdType( testTargetEdge.getTargetNode(), "test", "type2", null ) );
results = edges.toBlocking().getIterator();
@@ -1629,7 +2081,7 @@ public class GraphManagerIT {
@Test( expected = NullPointerException.class )
- public void invalidEdgeTypesWrite( ) {
+ public void invalidEdgeTypesWrite() {
final GraphManager em = emf.createEdgeManager( scope );
em.writeEdge( null );
@@ -1637,7 +2089,7 @@ public class GraphManagerIT {
@Test( expected = NullPointerException.class )
- public void invalidEdgeTypesDelete( ) {
+ public void invalidEdgeTypesDelete() {
final GraphManager em = emf.createEdgeManager( scope );
em.markEdge( null );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/62e650bc/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/test/util/EdgeTestUtils.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/test/util/EdgeTestUtils.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/test/util/EdgeTestUtils.java
index b2142e8..718fc70 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/test/util/EdgeTestUtils.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/test/util/EdgeTestUtils.java
@@ -152,6 +152,19 @@ public class EdgeTestUtils {
return new SimpleSearchByEdgeType( sourceId, type, maxVersion, SearchByEdgeType.Order.DESCENDING, last );
}
+ /**
+ *
+ * @param sourceId
+ * @param type
+ * @param maxVersion
+ * @param last
+ * @return
+ */
+ public static SearchByEdgeType createSearchByEdgeUnfiltered( final Id sourceId, final String type, final long maxVersion,
+ final Edge last ) {
+ return new SimpleSearchByEdgeType( sourceId, type, maxVersion, SearchByEdgeType.Order.DESCENDING, Optional.fromNullable( last ) , false );
+ }
+
/**
*
@@ -167,6 +180,20 @@ public class EdgeTestUtils {
return new SimpleSearchByIdType( sourceId, type, maxVersion, SearchByEdgeType.Order.DESCENDING, idType, Optional.fromNullable(last) );
}
+ /**
+ *
+ * @param sourceId
+ * @param type
+ * @param maxVersion
+ * @param idType
+ * @param last
+ * @return
+ */
+ public static SearchByIdType createSearchByEdgeAndIdUnfiltered( final Id sourceId, final String type, final long maxVersion,
+ final String idType, final Edge last ) {
+ return new SimpleSearchByIdType( sourceId, type, maxVersion, SearchByEdgeType.Order.DESCENDING, idType, Optional.fromNullable(last), false );
+ }
+
/**
*
[3/9] incubator-usergrid git commit: First pass at refactoring mark +
sweep
Posted by to...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3e2afe23/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTaskTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTaskTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTaskTest.java
deleted file mode 100644
index 8c26c5b..0000000
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTaskTest.java
+++ /dev/null
@@ -1,715 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.usergrid.persistence.collection.impl;
-
-
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.UUID;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Semaphore;
-
-import org.junit.AfterClass;
-import org.junit.Test;
-
-import org.apache.usergrid.persistence.collection.MvccLogEntry;
-import org.apache.usergrid.persistence.collection.event.EntityVersionDeleted;
-import org.apache.usergrid.persistence.collection.serialization.MvccLogEntrySerializationStrategy;
-import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
-import org.apache.usergrid.persistence.collection.serialization.UniqueValue;
-import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
-import org.apache.usergrid.persistence.collection.util.LogEntryMock;
-import org.apache.usergrid.persistence.collection.util.UniqueValueEntryMock;
-import org.apache.usergrid.persistence.collection.util.VersionGenerator;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
-import org.apache.usergrid.persistence.core.task.NamedTaskExecutorImpl;
-import org.apache.usergrid.persistence.core.task.TaskExecutor;
-import org.apache.usergrid.persistence.model.entity.Id;
-import org.apache.usergrid.persistence.model.entity.SimpleId;
-
-import com.google.common.util.concurrent.ListenableFuture;
-import com.netflix.astyanax.Keyspace;
-import com.netflix.astyanax.MutationBatch;
-import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
-
-import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.same;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.never;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
-
-
-/**
- * Cleanup task tests
- */
-public class EntityVersionCleanupTaskTest {
-
- private static final TaskExecutor taskExecutor = new NamedTaskExecutorImpl( "test", 4, 0 );
-
-
- @AfterClass
- public static void shutdown() {
- taskExecutor.shutdown();
- }
-
-
- @Test( timeout = 10000 )
- public void noListenerOneVersion() throws Exception {
-
-
- final SerializationFig serializationFig = mock( SerializationFig.class );
-
- when( serializationFig.getBufferSize() ).thenReturn( 10 );
-
- final MvccLogEntrySerializationStrategy less = mock( MvccLogEntrySerializationStrategy.class );
-
-
- final UniqueValueSerializationStrategy uvss = mock( UniqueValueSerializationStrategy.class );
-
- final Keyspace keyspace = mock( Keyspace.class );
-
- final MutationBatch entityBatch = mock( MutationBatch.class );
-
- when( keyspace.prepareMutationBatch() ).thenReturn(
- mock( MutationBatch.class ) ) // don't care what happens to this one
- .thenReturn( entityBatch );
-
- // intentionally no events
- final Set<EntityVersionDeleted> listeners = new HashSet<EntityVersionDeleted>();
-
- final Id applicationId = new SimpleId( "application" );
-
- final ApplicationScope appScope = new ApplicationScopeImpl( applicationId );
-
- final Id entityId = new SimpleId( "user" );
-
- final List<UUID> versions = VersionGenerator.generateVersions( 2 );
-
- // mock up a single log entry for our first test
- final LogEntryMock logEntryMock = LogEntryMock.createLogEntryMock( less, appScope, entityId, versions );
-
-
- //get the version we're keeping, it's first in our list
- final UUID version = logEntryMock.getEntryAtIndex( 0 ).getVersion();
-
- //mock up unique version output
- final UniqueValueEntryMock uniqueValueEntryMock =
- UniqueValueEntryMock.createUniqueMock( uvss, appScope, entityId, versions );
-
-
- EntityVersionCleanupTask cleanupTask =
- new EntityVersionCleanupTask( serializationFig, less, uvss, keyspace, listeners, appScope, entityId,
- version, false );
-
- final MutationBatch newBatch = mock( MutationBatch.class );
-
-
- // set up returning a mutator
- when( uvss.delete( same( appScope ), any( UniqueValue.class ) ) ).thenReturn( newBatch );
-
- //return a new batch when it's called
- when( less.delete( same( appScope ), same( entityId ), any( UUID.class ) ) ).thenReturn( newBatch );
-
-
- cleanupTask.call();
-
-
- //get the second field, this should be deleted
- final UniqueValue oldUniqueField = uniqueValueEntryMock.getEntryAtIndex( 1 );
-
- final MvccLogEntry expectedDeletedEntry = logEntryMock.getEntryAtIndex( 1 );
-
-
- //verify delete was invoked
- verify( uvss ).delete( same( appScope ), same( oldUniqueField ) );
-
- //verify the delete was invoked
- verify( less ).delete( same( appScope ), same( entityId ), same( expectedDeletedEntry.getVersion() ) );
-
- // verify it was run
- verify( entityBatch ).execute();
- }
-
-
- /**
- * Tests the cleanup task on the first version created
- */
- @Test( timeout = 10000 )
- public void noListenerNoVersions() throws Exception {
-
-
- final SerializationFig serializationFig = mock( SerializationFig.class );
-
- when( serializationFig.getBufferSize() ).thenReturn( 10 );
-
- final MvccLogEntrySerializationStrategy less = mock( MvccLogEntrySerializationStrategy.class );
-
-
- final UniqueValueSerializationStrategy uvss = mock( UniqueValueSerializationStrategy.class );
-
- final Keyspace keyspace = mock( Keyspace.class );
-
- final MutationBatch entityBatch = mock( MutationBatch.class );
-
- when( keyspace.prepareMutationBatch() ).thenReturn(
- mock( MutationBatch.class ) ) // don't care what happens to this one
- .thenReturn( entityBatch );
-
- // intentionally no events
- final Set<EntityVersionDeleted> listeners = new HashSet<>();
-
- final Id applicationId = new SimpleId( "application" );
-
- final ApplicationScope appScope = new ApplicationScopeImpl( applicationId );
-
- final Id entityId = new SimpleId( "user" );
-
-
- final List<UUID> versions = VersionGenerator.generateVersions( 1 );
-
- // mock up a single log entry, with no other entries
- final LogEntryMock logEntryMock = LogEntryMock.createLogEntryMock( less, appScope, entityId, versions );
-
-
- //get the version we're keeping, it's first in our list
- final UUID version = logEntryMock.getEntryAtIndex( 0 ).getVersion();
-
- //mock up unique version output
- final UniqueValueEntryMock uniqueValueEntryMock =
- UniqueValueEntryMock.createUniqueMock( uvss, appScope, entityId, versions );
-
-
- EntityVersionCleanupTask cleanupTask =
- new EntityVersionCleanupTask( serializationFig, less, uvss, keyspace, listeners, appScope, entityId, version, false );
-
- final MutationBatch newBatch = mock( MutationBatch.class );
-
-
- // set up returning a mutator
- when( uvss.delete( same( appScope ), any( UniqueValue.class ) ) ).thenReturn( newBatch );
-
- //return a new batch when it's called
- when( less.delete( same( appScope ), same( entityId ), any( UUID.class ) ) ).thenReturn( newBatch );
-
-
- cleanupTask.call();
-
-
- //verify delete was never invoked
- verify( uvss, never() ).delete( any( ApplicationScope.class ), any( UniqueValue.class ) );
-
- //verify the delete was never invoked
- verify( less, never() ).delete( any( ApplicationScope.class ), any( Id.class ), any( UUID.class ) );
- }
-
-
- @Test( timeout = 10000 )
- public void singleListenerSingleVersion() throws Exception {
-
-
- //create a latch for the event listener, and add it to the list of events
- final int sizeToReturn = 1;
-
- final CountDownLatch latch = new CountDownLatch( sizeToReturn );
-
- final EntityVersionDeletedTest eventListener = new EntityVersionDeletedTest( latch );
-
- final Set<EntityVersionDeleted> listeners = new HashSet<>();
-
- listeners.add( eventListener );
-
-
- final SerializationFig serializationFig = mock( SerializationFig.class );
-
- when( serializationFig.getBufferSize() ).thenReturn( 10 );
-
- final MvccLogEntrySerializationStrategy less = mock( MvccLogEntrySerializationStrategy.class );
-
- final UniqueValueSerializationStrategy uvss = mock( UniqueValueSerializationStrategy.class );
-
- final Keyspace keyspace = mock( Keyspace.class );
-
- final MutationBatch entityBatch = mock( MutationBatch.class );
-
- when( keyspace.prepareMutationBatch() ).thenReturn(
- mock( MutationBatch.class ) ) // don't care what happens to this one
- .thenReturn( entityBatch );
-
-
- final Id applicationId = new SimpleId( "application" );
-
- final ApplicationScope appScope = new ApplicationScopeImpl( applicationId );
-
- final Id entityId = new SimpleId( "user" );
-
-
- final List<UUID> versions = VersionGenerator.generateVersions( 2 );
-
-
- // mock up a single log entry for our first test
- final LogEntryMock logEntryMock = LogEntryMock.createLogEntryMock( less, appScope, entityId, versions );
-
-
- //get the version we're keeping, it's first in our list
- final UUID version = logEntryMock.getEntryAtIndex( 0 ).getVersion();
-
- //mock up unique version output
- final UniqueValueEntryMock uniqueValueEntryMock =
- UniqueValueEntryMock.createUniqueMock( uvss, appScope, entityId, versions );
-
-
- EntityVersionCleanupTask cleanupTask =
- new EntityVersionCleanupTask( serializationFig, less, uvss, keyspace, listeners, appScope, entityId,
- version, false );
-
- final MutationBatch newBatch = mock( MutationBatch.class );
-
-
- // set up returning a mutator
- when( uvss.delete( same( appScope ), any( UniqueValue.class ) ) ).thenReturn( newBatch );
-
- //return a new batch when it's called
- when( less.delete( same( appScope ), same( entityId ), any( UUID.class ) ) ).thenReturn( newBatch );
-
-
- cleanupTask.call();
-
-
- //get the second field, this should be deleted
- final UniqueValue oldUniqueField = uniqueValueEntryMock.getEntryAtIndex( 1 );
-
- final MvccLogEntry expectedDeletedEntry = logEntryMock.getEntryAtIndex( 1 );
-
-
- //verify delete was invoked
- verify( uvss ).delete( same( appScope ), same( oldUniqueField ) );
-
- //verify the delete was invoked
- verify( less ).delete( same( appScope ), same( entityId ), same( expectedDeletedEntry.getVersion() ) );
-
- // verify it was run
- verify( entityBatch ).execute();
-
-
- //the latch was executed
- latch.await();
- }
-
-
- @Test//(timeout=10000)
- public void multipleListenerMultipleVersions() throws Exception {
-
- final SerializationFig serializationFig = mock( SerializationFig.class );
-
- when( serializationFig.getBufferSize() ).thenReturn( 10 );
-
-
- //create a latch for the event listener, and add it to the list of events
- final int sizeToReturn = 10;
-
- final CountDownLatch latch = new CountDownLatch( sizeToReturn / serializationFig.getBufferSize() * 3 );
-
- final EntityVersionDeletedTest listener1 = new EntityVersionDeletedTest( latch );
- final EntityVersionDeletedTest listener2 = new EntityVersionDeletedTest( latch );
- final EntityVersionDeletedTest listener3 = new EntityVersionDeletedTest( latch );
-
- final Set<EntityVersionDeleted> listeners = new HashSet<>();
-
- listeners.add( listener1 );
- listeners.add( listener2 );
- listeners.add( listener3 );
-
-
-
-
- final MvccLogEntrySerializationStrategy less = mock( MvccLogEntrySerializationStrategy.class );
-
- final UniqueValueSerializationStrategy uvss = mock( UniqueValueSerializationStrategy.class );
-
- final Keyspace keyspace = mock( Keyspace.class );
-
- final MutationBatch entityBatch = mock( MutationBatch.class );
-
- when( keyspace.prepareMutationBatch() ).thenReturn(
- mock( MutationBatch.class ) ) // don't care what happens to this one
- .thenReturn( entityBatch );
-
-
-
-
- final Id applicationId = new SimpleId( "application" );
-
- final ApplicationScope appScope = new ApplicationScopeImpl( applicationId );
-
- final Id entityId = new SimpleId( "user" );
-
- final List<UUID> versions = VersionGenerator.generateVersions( 2 );
-
-
- // mock up a single log entry for our first test
- final LogEntryMock logEntryMock = LogEntryMock.createLogEntryMock( less, appScope, entityId, versions );
-
-
- //get the version we're keeping, it's first in our list
- final UUID version = logEntryMock.getEntryAtIndex( 0 ).getVersion();
-
- //mock up unique version output
- final UniqueValueEntryMock uniqueValueEntryMock =
- UniqueValueEntryMock.createUniqueMock( uvss, appScope, entityId, versions );
-
-
- EntityVersionCleanupTask cleanupTask =
- new EntityVersionCleanupTask( serializationFig, less, uvss, keyspace, listeners, appScope, entityId,
- version, false );
-
- final MutationBatch newBatch = mock( MutationBatch.class );
-
-
- // set up returning a mutator
- when( uvss.delete( same( appScope ), any( UniqueValue.class ) ) ).thenReturn( newBatch );
-
- //return a new batch when it's called
- when( less.delete( same( appScope ), same( entityId ), any( UUID.class ) ) ).thenReturn( newBatch );
-
-
- cleanupTask.call();
-
-
- //get the second field, this should be deleted
- final UniqueValue oldUniqueField = uniqueValueEntryMock.getEntryAtIndex( 1 );
-
- final MvccLogEntry expectedDeletedEntry = logEntryMock.getEntryAtIndex( 1 );
-
-
- //verify delete was invoked
- verify( uvss ).delete( same( appScope ), same( oldUniqueField ) );
-
- //verify the delete was invoked
- verify( less ).delete( same( appScope ), same( entityId ), same( expectedDeletedEntry.getVersion() ) );
-
- // verify it was run
- verify( entityBatch ).execute();
-
-
- //the latch was executed
- latch.await();
-
- //we deleted the version
- //verify we deleted everything
- //verify delete was invoked
- verify( uvss ).delete( same( appScope ), same( oldUniqueField ) );
-
- //verify the delete was invoked
- verify( less ).delete( same( appScope ), same( entityId ), same( expectedDeletedEntry.getVersion() ) );
-
- // verify it was run
- verify( entityBatch ).execute();
-
- //the latch was executed
- latch.await();
- }
-
-
- /**
- * Tests what happens when our listeners are VERY slow
- */
-// @Ignore( "Test is a work in progress" )
- @Test( timeout = 10000 )
- public void multipleListenerMultipleVersionsNoThreadsToRun()
- throws ExecutionException, InterruptedException, ConnectionException {
-
-
- final SerializationFig serializationFig = mock( SerializationFig.class );
-
- when( serializationFig.getBufferSize() ).thenReturn( 10 );
-
-
- //create a latch for the event listener, and add it to the list of events
- final int sizeToReturn = 10;
-
-
- final int listenerCount = 5;
-
- final CountDownLatch latch =
- new CountDownLatch( sizeToReturn / serializationFig.getBufferSize() * listenerCount );
- final Semaphore waitSemaphore = new Semaphore( 0 );
-
-
- final SlowListener listener1 = new SlowListener( latch, waitSemaphore );
- final SlowListener listener2 = new SlowListener( latch, waitSemaphore );
- final SlowListener listener3 = new SlowListener( latch, waitSemaphore );
- final SlowListener listener4 = new SlowListener( latch, waitSemaphore );
- final SlowListener listener5 = new SlowListener( latch, waitSemaphore );
-
- final Set<EntityVersionDeleted> listeners = new HashSet<EntityVersionDeleted>();
-
- listeners.add( listener1 );
- listeners.add( listener2 );
- listeners.add( listener3 );
- listeners.add( listener4 );
- listeners.add( listener5 );
-
-
- final MvccLogEntrySerializationStrategy less = mock( MvccLogEntrySerializationStrategy.class );
-
- final UniqueValueSerializationStrategy uvss = mock( UniqueValueSerializationStrategy.class );
-
- final Keyspace keyspace = mock( Keyspace.class );
-
- final MutationBatch entityBatch = mock( MutationBatch.class );
-
- when( keyspace.prepareMutationBatch() ).thenReturn(
- mock( MutationBatch.class ) ) // don't care what happens to this one
- .thenReturn( entityBatch );
-
-
- final Id applicationId = new SimpleId( "application" );
-
- final ApplicationScope appScope = new ApplicationScopeImpl( applicationId );
-
- final Id entityId = new SimpleId( "user" );
-
-
- final List<UUID> versions = VersionGenerator.generateVersions( 2 );
-
- // mock up a single log entry for our first test
- final LogEntryMock logEntryMock = LogEntryMock.createLogEntryMock( less, appScope, entityId, versions );
-
-
- //get the version we're keeping, it's first in our list
- final UUID version = logEntryMock.getEntryAtIndex( 0 ).getVersion();
-
-
- //mock up unique version output
- final UniqueValueEntryMock uniqueValueEntryMock =
- UniqueValueEntryMock.createUniqueMock( uvss, appScope, entityId, versions );
-
-
- EntityVersionCleanupTask cleanupTask =
- new EntityVersionCleanupTask( serializationFig, less, uvss, keyspace, listeners, appScope, entityId,
- version, false);
-
- final MutationBatch newBatch = mock( MutationBatch.class );
-
-
- // set up returning a mutator
- when( uvss.delete( same( appScope ), any( UniqueValue.class ) ) ).thenReturn( newBatch );
-
- //return a new batch when it's called
- when( less.delete( same( appScope ), same( entityId ), any( UUID.class ) ) ).thenReturn( newBatch );
-
-
- //start the task
- ListenableFuture<Void> future = taskExecutor.submit( cleanupTask );
-
- /**
- * While we're not done, release latches every 200 ms
- */
- while ( !future.isDone() ) {
- Thread.sleep( 200 );
- waitSemaphore.release( listenerCount );
- }
-
- //wait for the task
- future.get();
-
-
- //get the second field, this should be deleted
- final UniqueValue oldUniqueField = uniqueValueEntryMock.getEntryAtIndex( 1 );
-
- final MvccLogEntry expectedDeletedEntry = logEntryMock.getEntryAtIndex( 1 );
-
-
- //verify delete was invoked
- verify( uvss ).delete( same( appScope ), same( oldUniqueField ) );
-
- //verify the delete was invoked
- verify( less ).delete( same( appScope ), same( entityId ), same( expectedDeletedEntry.getVersion() ) );
-
- // verify it was run
- verify( entityBatch ).execute();
-
-
- //the latch was executed
- latch.await();
-
- //we deleted the version
- //verify we deleted everything
- //verify delete was invoked
- verify( uvss ).delete( same( appScope ), same( oldUniqueField ) );
-
- //verify the delete was invoked
- verify( less ).delete( same( appScope ), same( entityId ), same( expectedDeletedEntry.getVersion() ) );
-
- // verify it was run
- verify( entityBatch ).execute();
-
- //the latch was executed
- latch.await();
-
-
- //the latch was executed
- latch.await();
- }
-
-
- /**
- * Tests that our task will run in the caller if there's no threads, ensures that the task runs
- */
- @Test( timeout = 10000 )
- public void singleListenerSingleVersionRejected()
- throws ExecutionException, InterruptedException, ConnectionException {
-
-
-
- //create a latch for the event listener, and add it to the list of events
- final int sizeToReturn = 1;
-
- final CountDownLatch latch = new CountDownLatch( sizeToReturn );
-
- final EntityVersionDeletedTest eventListener = new EntityVersionDeletedTest( latch );
-
- final Set<EntityVersionDeleted> listeners = new HashSet<>();
-
- listeners.add( eventListener );
-
-
- final SerializationFig serializationFig = mock( SerializationFig.class );
-
- when( serializationFig.getBufferSize() ).thenReturn( 10 );
-
- final MvccLogEntrySerializationStrategy less = mock( MvccLogEntrySerializationStrategy.class );
-
- final UniqueValueSerializationStrategy uvss = mock( UniqueValueSerializationStrategy.class );
-
- final Keyspace keyspace = mock( Keyspace.class );
-
- final MutationBatch entityBatch = mock( MutationBatch.class );
-
- when( keyspace.prepareMutationBatch() ).thenReturn(
- mock( MutationBatch.class ) ) // don't care what happens to this one
- .thenReturn( entityBatch );
-
-
- final Id applicationId = new SimpleId( "application" );
-
- final ApplicationScope appScope = new ApplicationScopeImpl( applicationId );
-
- final Id entityId = new SimpleId( "user" );
-
-
- final List<UUID> versions = VersionGenerator.generateVersions( 2 );
-
-
- // mock up a single log entry for our first test
- final LogEntryMock logEntryMock = LogEntryMock.createLogEntryMock( less, appScope, entityId, versions );
-
-
- //get the version we're keeping, it's first in our list
- final UUID version = logEntryMock.getEntryAtIndex( 0 ).getVersion();
-
- //mock up unique version output
- final UniqueValueEntryMock uniqueValueEntryMock =
- UniqueValueEntryMock.createUniqueMock( uvss, appScope, entityId, versions );
-
-
- EntityVersionCleanupTask cleanupTask =
- new EntityVersionCleanupTask( serializationFig, less, uvss, keyspace, listeners, appScope, entityId,
- version, false );
-
- final MutationBatch newBatch = mock( MutationBatch.class );
-
-
- // set up returning a mutator
- when( uvss.delete( same( appScope ), any( UniqueValue.class ) ) ).thenReturn( newBatch );
-
- //return a new batch when it's called
- when( less.delete( same( appScope ), same( entityId ), any( UUID.class ) ) ).thenReturn( newBatch );
-
-
- cleanupTask.rejected();
-
-
- //get the second field, this should be deleted
- final UniqueValue oldUniqueField = uniqueValueEntryMock.getEntryAtIndex( 1 );
-
- final MvccLogEntry expectedDeletedEntry = logEntryMock.getEntryAtIndex( 1 );
-
-
- //verify delete was invoked
- verify( uvss ).delete( same( appScope ), same( oldUniqueField ) );
-
- //verify the delete was invoked
- verify( less ).delete( same( appScope ), same( entityId ), same( expectedDeletedEntry.getVersion() ) );
-
- // verify it was run
- verify( entityBatch ).execute();
-
-
- //the latch was executed
- latch.await();
- }
-
-
- private static class EntityVersionDeletedTest implements EntityVersionDeleted {
- final CountDownLatch invocationLatch;
-
-
- private EntityVersionDeletedTest( final CountDownLatch invocationLatch ) {
- this.invocationLatch = invocationLatch;
- }
-
-
- @Override
- public void versionDeleted( final ApplicationScope scope, final Id entityId,
- final List<MvccLogEntry> entityVersion ) {
- invocationLatch.countDown();
- }
- }
-
-
- private static class SlowListener extends EntityVersionDeletedTest {
- final Semaphore blockLatch;
-
-
- private SlowListener( final CountDownLatch invocationLatch, final Semaphore blockLatch ) {
- super( invocationLatch );
- this.blockLatch = blockLatch;
- }
-
-
- @Override
- public void versionDeleted( final ApplicationScope scope, final Id entityId,
- final List<MvccLogEntry> entityVersion ) {
-
- //wait for unblock to happen before counting down invocation latches
- try {
- blockLatch.acquire();
- }
- catch ( InterruptedException e ) {
- throw new RuntimeException( e );
- }
- super.versionDeleted( scope, entityId, entityVersion );
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3e2afe23/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTaskTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTaskTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTaskTest.java
deleted file mode 100644
index e993fad..0000000
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTaskTest.java
+++ /dev/null
@@ -1,241 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. The ASF licenses this file to You
- * under the Apache License, Version 2.0 (the "License"); you may not
- * use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License. For additional information regarding
- * copyright in this work, please see the NOTICE file in the top level
- * directory of this distribution.
- */
-package org.apache.usergrid.persistence.collection.impl;
-
-
-import java.util.Iterator;
-import java.util.Set;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.ExecutionException;
-
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.Test;
-
-import org.apache.usergrid.persistence.collection.event.EntityVersionCreated;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
-import org.apache.usergrid.persistence.core.task.NamedTaskExecutorImpl;
-import org.apache.usergrid.persistence.core.task.TaskExecutor;
-import org.apache.usergrid.persistence.model.entity.Entity;
-import org.apache.usergrid.persistence.model.entity.Id;
-import org.apache.usergrid.persistence.model.entity.SimpleId;
-
-import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
-
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
-
-
-/**
- * Created task tests.
- */
-public class EntityVersionCreatedTaskTest {
-
- private static final TaskExecutor taskExecutor = new NamedTaskExecutorImpl( "test", 4, 0 );
-
- @AfterClass
- public static void shutdown() {
- taskExecutor.shutdown();
- }
-
-
- @Test(timeout=10000)
- public void noListener()
- throws ExecutionException, InterruptedException, ConnectionException {
-
- // create a latch for the event listener, and add it to the list of events
-
- final Set<EntityVersionCreated> listeners = mock( Set.class );
-
- when ( listeners.size()).thenReturn( 0 );
-
- final Id applicationId = new SimpleId( "application" );
-
- final ApplicationScope appScope = new ApplicationScopeImpl(applicationId);
-
- final Id entityId = new SimpleId( "user" );
- final Entity entity = new Entity( entityId );
-
- // start the task
-
- EntityVersionCreatedTask entityVersionCreatedTask =
- new EntityVersionCreatedTask( appScope, listeners, entity);
-
- try {
- entityVersionCreatedTask.call();
- }catch(Exception e){
- Assert.fail(e.getMessage());
- }
-
-
- // wait for the task
- // future.get();
-
- //mocked listener makes sure that the task is called
- verify( listeners ).size();
-
- }
- @Test(timeout=10000)
- public void oneListener()
- throws ExecutionException, InterruptedException, ConnectionException {
-
- // create a latch for the event listener, and add it to the list of events
-
- final int sizeToReturn = 1;
-
- final CountDownLatch latch = new CountDownLatch( sizeToReturn );
-
- final EntityVersionCreatedTest eventListener = new EntityVersionCreatedTest(latch);
-
- final Set<EntityVersionCreated> listeners = mock( Set.class );
- final Iterator<EntityVersionCreated> helper = mock(Iterator.class);
-
- when ( listeners.size()).thenReturn( 1 );
- when ( listeners.iterator()).thenReturn( helper );
- when ( helper.next() ).thenReturn( eventListener );
-
- final Id applicationId = new SimpleId( "application" );
-
- final ApplicationScope appScope = new ApplicationScopeImpl(applicationId);
-
- final Id entityId = new SimpleId( "user" );
- final Entity entity = new Entity( entityId );
-
- // start the task
-
- EntityVersionCreatedTask entityVersionCreatedTask =
- new EntityVersionCreatedTask( appScope, listeners, entity);
-
- try {
- entityVersionCreatedTask.call();
- }catch(Exception e){
-
- Assert.fail(e.getMessage());
- }
- //mocked listener makes sure that the task is called
- verify( listeners ).size();
- verify( listeners ).iterator();
- verify( helper ).next();
-
- }
-
- @Test(timeout=10000)
- public void multipleListener()
- throws ExecutionException, InterruptedException, ConnectionException {
-
- final int sizeToReturn = 3;
-
- final Set<EntityVersionCreated> listeners = mock( Set.class );
- final Iterator<EntityVersionCreated> helper = mock(Iterator.class);
-
- when ( listeners.size()).thenReturn( 3 );
- when ( listeners.iterator()).thenReturn( helper );
-
- final Id applicationId = new SimpleId( "application" );
-
- final ApplicationScope appScope = new ApplicationScopeImpl(applicationId);
-
- final Id entityId = new SimpleId( "user" );
- final Entity entity = new Entity( entityId );
-
- // start the task
-
- EntityVersionCreatedTask entityVersionCreatedTask =
- new EntityVersionCreatedTask( appScope, listeners, entity);
-
- final CountDownLatch latch = new CountDownLatch( sizeToReturn );
-
- final EntityVersionCreatedTest listener1 = new EntityVersionCreatedTest(latch);
- final EntityVersionCreatedTest listener2 = new EntityVersionCreatedTest(latch);
- final EntityVersionCreatedTest listener3 = new EntityVersionCreatedTest(latch);
-
- when ( helper.next() ).thenReturn( listener1,listener2,listener3);
-
- try {
- entityVersionCreatedTask.call();
- }catch(Exception e){
- ;
- }
- //ListenableFuture<Void> future = taskExecutor.submit( entityVersionCreatedTask );
-
- //wait for the task
- //intentionally fails due to difficulty mocking observable
-
- //mocked listener makes sure that the task is called
- verify( listeners ).size();
- //verifies that the observable made listener iterate.
- verify( listeners ).iterator();
- }
-
- @Test(timeout=10000)
- public void oneListenerRejected()
- throws ExecutionException, InterruptedException, ConnectionException {
-
- // create a latch for the event listener, and add it to the list of events
-
- final TaskExecutor taskExecutor = new NamedTaskExecutorImpl( "test", 0, 0 );
-
- final int sizeToReturn = 1;
-
- final CountDownLatch latch = new CountDownLatch( sizeToReturn );
-
- final EntityVersionCreatedTest eventListener = new EntityVersionCreatedTest(latch);
-
- final Set<EntityVersionCreated> listeners = mock( Set.class );
- final Iterator<EntityVersionCreated> helper = mock(Iterator.class);
-
- when ( listeners.size()).thenReturn( 1 );
- when ( listeners.iterator()).thenReturn( helper );
- when ( helper.next() ).thenReturn( eventListener );
-
- final Id applicationId = new SimpleId( "application" );
-
- final ApplicationScope appScope = new ApplicationScopeImpl(applicationId);
-
- final Id entityId = new SimpleId( "user" );
- final Entity entity = new Entity( entityId );
-
- // start the task
-
- EntityVersionCreatedTask entityVersionCreatedTask =
- new EntityVersionCreatedTask( appScope, listeners, entity);
-
- entityVersionCreatedTask.rejected();
-
- //mocked listener makes sure that the task is called
- verify( listeners ).size();
- verify( listeners ).iterator();
- verify( helper ).next();
-
- }
-
- private static class EntityVersionCreatedTest implements EntityVersionCreated {
- final CountDownLatch invocationLatch;
-
- private EntityVersionCreatedTest( final CountDownLatch invocationLatch) {
- this.invocationLatch = invocationLatch;
- }
-
- @Override
- public void versionCreated( final ApplicationScope scope, final Entity entity ) {
- invocationLatch.countDown();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3e2afe23/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/UniqueCleanupTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/UniqueCleanupTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/UniqueCleanupTest.java
new file mode 100644
index 0000000..c1f76f2
--- /dev/null
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/UniqueCleanupTest.java
@@ -0,0 +1,712 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence.collection.mvcc.stage.delete;
+
+
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Semaphore;
+
+import org.junit.AfterClass;
+import org.junit.Test;
+
+import org.apache.usergrid.persistence.collection.MvccLogEntry;
+import org.apache.usergrid.persistence.collection.serialization.MvccLogEntrySerializationStrategy;
+import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
+import org.apache.usergrid.persistence.collection.serialization.UniqueValue;
+import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
+import org.apache.usergrid.persistence.collection.util.LogEntryMock;
+import org.apache.usergrid.persistence.collection.util.UniqueValueEntryMock;
+import org.apache.usergrid.persistence.collection.util.VersionGenerator;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
+import org.apache.usergrid.persistence.model.entity.Id;
+import org.apache.usergrid.persistence.model.entity.SimpleId;
+
+import com.google.common.util.concurrent.ListenableFuture;
+import com.netflix.astyanax.Keyspace;
+import com.netflix.astyanax.MutationBatch;
+import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
+
+import static org.junit.Assert.*;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.same;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+
+public class UniqueCleanupTest {
+
+//
+// private static final TaskExecutor taskExecutor = new NamedTaskExecutorImpl( "test", 4, 0 );
+//
+//
+// @AfterClass
+// public static void shutdown() {
+// taskExecutor.shutdown();
+// }
+//
+//
+// @Test( timeout = 10000 )
+// public void noListenerOneVersion() throws Exception {
+//
+//
+// final SerializationFig serializationFig = mock( SerializationFig.class );
+//
+// when( serializationFig.getBufferSize() ).thenReturn( 10 );
+//
+// final MvccLogEntrySerializationStrategy less = mock( MvccLogEntrySerializationStrategy.class );
+//
+//
+// final UniqueValueSerializationStrategy uvss = mock( UniqueValueSerializationStrategy.class );
+//
+// final Keyspace keyspace = mock( Keyspace.class );
+//
+// final MutationBatch entityBatch = mock( MutationBatch.class );
+//
+// when( keyspace.prepareMutationBatch() ).thenReturn(
+// mock( MutationBatch.class ) ) // don't care what happens to this one
+// .thenReturn( entityBatch );
+//
+// // intentionally no events
+// final Set<EntityVersionDeleted> listeners = new HashSet<EntityVersionDeleted>();
+//
+// final Id applicationId = new SimpleId( "application" );
+//
+// final ApplicationScope appScope = new ApplicationScopeImpl( applicationId );
+//
+// final Id entityId = new SimpleId( "user" );
+//
+// final List<UUID> versions = VersionGenerator.generateVersions( 2 );
+//
+// // mock up a single log entry for our first test
+// final LogEntryMock logEntryMock = LogEntryMock.createLogEntryMock( less, appScope, entityId, versions );
+//
+//
+// //get the version we're keeping, it's first in our list
+// final UUID version = logEntryMock.getEntryAtIndex( 0 ).getVersion();
+//
+// //mock up unique version output
+// final UniqueValueEntryMock uniqueValueEntryMock =
+// UniqueValueEntryMock.createUniqueMock( uvss, appScope, entityId, versions );
+//
+//
+// EntityVersionCleanupTask cleanupTask =
+// new EntityVersionCleanupTask( serializationFig, less, uvss, keyspace, listeners, appScope, entityId,
+// version, false );
+//
+// final MutationBatch newBatch = mock( MutationBatch.class );
+//
+//
+// // set up returning a mutator
+// when( uvss.delete( same( appScope ), any( UniqueValue.class ) ) ).thenReturn( newBatch );
+//
+// //return a new batch when it's called
+// when( less.delete( same( appScope ), same( entityId ), any( UUID.class ) ) ).thenReturn( newBatch );
+//
+//
+// cleanupTask.call();
+//
+//
+// //get the second field, this should be deleted
+// final UniqueValue oldUniqueField = uniqueValueEntryMock.getEntryAtIndex( 1 );
+//
+// final MvccLogEntry expectedDeletedEntry = logEntryMock.getEntryAtIndex( 1 );
+//
+//
+// //verify delete was invoked
+// verify( uvss ).delete( same( appScope ), same( oldUniqueField ) );
+//
+// //verify the delete was invoked
+// verify( less ).delete( same( appScope ), same( entityId ), same( expectedDeletedEntry.getVersion() ) );
+//
+// // verify it was run
+// verify( entityBatch ).execute();
+// }
+//
+//
+// /**
+// * Tests the cleanup task on the first version created
+// */
+// @Test( timeout = 10000 )
+// public void noListenerNoVersions() throws Exception {
+//
+//
+// final SerializationFig serializationFig = mock( SerializationFig.class );
+//
+// when( serializationFig.getBufferSize() ).thenReturn( 10 );
+//
+// final MvccLogEntrySerializationStrategy less = mock( MvccLogEntrySerializationStrategy.class );
+//
+//
+// final UniqueValueSerializationStrategy uvss = mock( UniqueValueSerializationStrategy.class );
+//
+// final Keyspace keyspace = mock( Keyspace.class );
+//
+// final MutationBatch entityBatch = mock( MutationBatch.class );
+//
+// when( keyspace.prepareMutationBatch() ).thenReturn(
+// mock( MutationBatch.class ) ) // don't care what happens to this one
+// .thenReturn( entityBatch );
+//
+// // intentionally no events
+// final Set<EntityVersionDeleted> listeners = new HashSet<>();
+//
+// final Id applicationId = new SimpleId( "application" );
+//
+// final ApplicationScope appScope = new ApplicationScopeImpl( applicationId );
+//
+// final Id entityId = new SimpleId( "user" );
+//
+//
+// final List<UUID> versions = VersionGenerator.generateVersions( 1 );
+//
+// // mock up a single log entry, with no other entries
+// final LogEntryMock logEntryMock = LogEntryMock.createLogEntryMock( less, appScope, entityId, versions );
+//
+//
+// //get the version we're keeping, it's first in our list
+// final UUID version = logEntryMock.getEntryAtIndex( 0 ).getVersion();
+//
+// //mock up unique version output
+// final UniqueValueEntryMock uniqueValueEntryMock =
+// UniqueValueEntryMock.createUniqueMock( uvss, appScope, entityId, versions );
+//
+//
+// EntityVersionCleanupTask cleanupTask =
+// new EntityVersionCleanupTask( serializationFig, less, uvss, keyspace, listeners, appScope, entityId, version, false );
+//
+// final MutationBatch newBatch = mock( MutationBatch.class );
+//
+//
+// // set up returning a mutator
+// when( uvss.delete( same( appScope ), any( UniqueValue.class ) ) ).thenReturn( newBatch );
+//
+// //return a new batch when it's called
+// when( less.delete( same( appScope ), same( entityId ), any( UUID.class ) ) ).thenReturn( newBatch );
+//
+//
+// cleanupTask.call();
+//
+//
+// //verify delete was never invoked
+// verify( uvss, never() ).delete( any( ApplicationScope.class ), any( UniqueValue.class ) );
+//
+// //verify the delete was never invoked
+// verify( less, never() ).delete( any( ApplicationScope.class ), any( Id.class ), any( UUID.class ) );
+// }
+//
+//
+// @Test( timeout = 10000 )
+// public void singleListenerSingleVersion() throws Exception {
+//
+//
+// //create a latch for the event listener, and add it to the list of events
+// final int sizeToReturn = 1;
+//
+// final CountDownLatch latch = new CountDownLatch( sizeToReturn );
+//
+// final EntityVersionDeletedTest eventListener = new EntityVersionDeletedTest( latch );
+//
+// final Set<EntityVersionDeleted> listeners = new HashSet<>();
+//
+// listeners.add( eventListener );
+//
+//
+// final SerializationFig serializationFig = mock( SerializationFig.class );
+//
+// when( serializationFig.getBufferSize() ).thenReturn( 10 );
+//
+// final MvccLogEntrySerializationStrategy less = mock( MvccLogEntrySerializationStrategy.class );
+//
+// final UniqueValueSerializationStrategy uvss = mock( UniqueValueSerializationStrategy.class );
+//
+// final Keyspace keyspace = mock( Keyspace.class );
+//
+// final MutationBatch entityBatch = mock( MutationBatch.class );
+//
+// when( keyspace.prepareMutationBatch() ).thenReturn(
+// mock( MutationBatch.class ) ) // don't care what happens to this one
+// .thenReturn( entityBatch );
+//
+//
+// final Id applicationId = new SimpleId( "application" );
+//
+// final ApplicationScope appScope = new ApplicationScopeImpl( applicationId );
+//
+// final Id entityId = new SimpleId( "user" );
+//
+//
+// final List<UUID> versions = VersionGenerator.generateVersions( 2 );
+//
+//
+// // mock up a single log entry for our first test
+// final LogEntryMock logEntryMock = LogEntryMock.createLogEntryMock( less, appScope, entityId, versions );
+//
+//
+// //get the version we're keeping, it's first in our list
+// final UUID version = logEntryMock.getEntryAtIndex( 0 ).getVersion();
+//
+// //mock up unique version output
+// final UniqueValueEntryMock uniqueValueEntryMock =
+// UniqueValueEntryMock.createUniqueMock( uvss, appScope, entityId, versions );
+//
+//
+// EntityVersionCleanupTask cleanupTask =
+// new EntityVersionCleanupTask( serializationFig, less, uvss, keyspace, listeners, appScope, entityId,
+// version, false );
+//
+// final MutationBatch newBatch = mock( MutationBatch.class );
+//
+//
+// // set up returning a mutator
+// when( uvss.delete( same( appScope ), any( UniqueValue.class ) ) ).thenReturn( newBatch );
+//
+// //return a new batch when it's called
+// when( less.delete( same( appScope ), same( entityId ), any( UUID.class ) ) ).thenReturn( newBatch );
+//
+//
+// cleanupTask.call();
+//
+//
+// //get the second field, this should be deleted
+// final UniqueValue oldUniqueField = uniqueValueEntryMock.getEntryAtIndex( 1 );
+//
+// final MvccLogEntry expectedDeletedEntry = logEntryMock.getEntryAtIndex( 1 );
+//
+//
+// //verify delete was invoked
+// verify( uvss ).delete( same( appScope ), same( oldUniqueField ) );
+//
+// //verify the delete was invoked
+// verify( less ).delete( same( appScope ), same( entityId ), same( expectedDeletedEntry.getVersion() ) );
+//
+// // verify it was run
+// verify( entityBatch ).execute();
+//
+//
+// //the latch was executed
+// latch.await();
+// }
+//
+//
+// @Test//(timeout=10000)
+// public void multipleListenerMultipleVersions() throws Exception {
+//
+// final SerializationFig serializationFig = mock( SerializationFig.class );
+//
+// when( serializationFig.getBufferSize() ).thenReturn( 10 );
+//
+//
+// //create a latch for the event listener, and add it to the list of events
+// final int sizeToReturn = 10;
+//
+// final CountDownLatch latch = new CountDownLatch( sizeToReturn / serializationFig.getBufferSize() * 3 );
+//
+// final EntityVersionDeletedTest listener1 = new EntityVersionDeletedTest( latch );
+// final EntityVersionDeletedTest listener2 = new EntityVersionDeletedTest( latch );
+// final EntityVersionDeletedTest listener3 = new EntityVersionDeletedTest( latch );
+//
+// final Set<EntityVersionDeleted> listeners = new HashSet<>();
+//
+// listeners.add( listener1 );
+// listeners.add( listener2 );
+// listeners.add( listener3 );
+//
+//
+//
+//
+// final MvccLogEntrySerializationStrategy less = mock( MvccLogEntrySerializationStrategy.class );
+//
+// final UniqueValueSerializationStrategy uvss = mock( UniqueValueSerializationStrategy.class );
+//
+// final Keyspace keyspace = mock( Keyspace.class );
+//
+// final MutationBatch entityBatch = mock( MutationBatch.class );
+//
+// when( keyspace.prepareMutationBatch() ).thenReturn(
+// mock( MutationBatch.class ) ) // don't care what happens to this one
+// .thenReturn( entityBatch );
+//
+//
+//
+//
+// final Id applicationId = new SimpleId( "application" );
+//
+// final ApplicationScope appScope = new ApplicationScopeImpl( applicationId );
+//
+// final Id entityId = new SimpleId( "user" );
+//
+// final List<UUID> versions = VersionGenerator.generateVersions( 2 );
+//
+//
+// // mock up a single log entry for our first test
+// final LogEntryMock logEntryMock = LogEntryMock.createLogEntryMock( less, appScope, entityId, versions );
+//
+//
+// //get the version we're keeping, it's first in our list
+// final UUID version = logEntryMock.getEntryAtIndex( 0 ).getVersion();
+//
+// //mock up unique version output
+// final UniqueValueEntryMock uniqueValueEntryMock =
+// UniqueValueEntryMock.createUniqueMock( uvss, appScope, entityId, versions );
+//
+//
+// EntityVersionCleanupTask cleanupTask =
+// new EntityVersionCleanupTask( serializationFig, less, uvss, keyspace, listeners, appScope, entityId,
+// version, false );
+//
+// final MutationBatch newBatch = mock( MutationBatch.class );
+//
+//
+// // set up returning a mutator
+// when( uvss.delete( same( appScope ), any( UniqueValue.class ) ) ).thenReturn( newBatch );
+//
+// //return a new batch when it's called
+// when( less.delete( same( appScope ), same( entityId ), any( UUID.class ) ) ).thenReturn( newBatch );
+//
+//
+// cleanupTask.call();
+//
+//
+// //get the second field, this should be deleted
+// final UniqueValue oldUniqueField = uniqueValueEntryMock.getEntryAtIndex( 1 );
+//
+// final MvccLogEntry expectedDeletedEntry = logEntryMock.getEntryAtIndex( 1 );
+//
+//
+// //verify delete was invoked
+// verify( uvss ).delete( same( appScope ), same( oldUniqueField ) );
+//
+// //verify the delete was invoked
+// verify( less ).delete( same( appScope ), same( entityId ), same( expectedDeletedEntry.getVersion() ) );
+//
+// // verify it was run
+// verify( entityBatch ).execute();
+//
+//
+// //the latch was executed
+// latch.await();
+//
+// //we deleted the version
+// //verify we deleted everything
+// //verify delete was invoked
+// verify( uvss ).delete( same( appScope ), same( oldUniqueField ) );
+//
+// //verify the delete was invoked
+// verify( less ).delete( same( appScope ), same( entityId ), same( expectedDeletedEntry.getVersion() ) );
+//
+// // verify it was run
+// verify( entityBatch ).execute();
+//
+// //the latch was executed
+// latch.await();
+// }
+//
+//
+// /**
+// * Tests what happens when our listeners are VERY slow
+// */
+//// @Ignore( "Test is a work in progress" )
+// @Test( timeout = 10000 )
+// public void multipleListenerMultipleVersionsNoThreadsToRun()
+// throws ExecutionException, InterruptedException, ConnectionException {
+//
+//
+// final SerializationFig serializationFig = mock( SerializationFig.class );
+//
+// when( serializationFig.getBufferSize() ).thenReturn( 10 );
+//
+//
+// //create a latch for the event listener, and add it to the list of events
+// final int sizeToReturn = 10;
+//
+//
+// final int listenerCount = 5;
+//
+// final CountDownLatch latch =
+// new CountDownLatch( sizeToReturn / serializationFig.getBufferSize() * listenerCount );
+// final Semaphore waitSemaphore = new Semaphore( 0 );
+//
+//
+// final SlowListener listener1 = new SlowListener( latch, waitSemaphore );
+// final SlowListener listener2 = new SlowListener( latch, waitSemaphore );
+// final SlowListener listener3 = new SlowListener( latch, waitSemaphore );
+// final SlowListener listener4 = new SlowListener( latch, waitSemaphore );
+// final SlowListener listener5 = new SlowListener( latch, waitSemaphore );
+//
+// final Set<EntityVersionDeleted> listeners = new HashSet<EntityVersionDeleted>();
+//
+// listeners.add( listener1 );
+// listeners.add( listener2 );
+// listeners.add( listener3 );
+// listeners.add( listener4 );
+// listeners.add( listener5 );
+//
+//
+// final MvccLogEntrySerializationStrategy less = mock( MvccLogEntrySerializationStrategy.class );
+//
+// final UniqueValueSerializationStrategy uvss = mock( UniqueValueSerializationStrategy.class );
+//
+// final Keyspace keyspace = mock( Keyspace.class );
+//
+// final MutationBatch entityBatch = mock( MutationBatch.class );
+//
+// when( keyspace.prepareMutationBatch() ).thenReturn(
+// mock( MutationBatch.class ) ) // don't care what happens to this one
+// .thenReturn( entityBatch );
+//
+//
+// final Id applicationId = new SimpleId( "application" );
+//
+// final ApplicationScope appScope = new ApplicationScopeImpl( applicationId );
+//
+// final Id entityId = new SimpleId( "user" );
+//
+//
+// final List<UUID> versions = VersionGenerator.generateVersions( 2 );
+//
+// // mock up a single log entry for our first test
+// final LogEntryMock logEntryMock = LogEntryMock.createLogEntryMock( less, appScope, entityId, versions );
+//
+//
+// //get the version we're keeping, it's first in our list
+// final UUID version = logEntryMock.getEntryAtIndex( 0 ).getVersion();
+//
+//
+// //mock up unique version output
+// final UniqueValueEntryMock uniqueValueEntryMock =
+// UniqueValueEntryMock.createUniqueMock( uvss, appScope, entityId, versions );
+//
+//
+// EntityVersionCleanupTask cleanupTask =
+// new EntityVersionCleanupTask( serializationFig, less, uvss, keyspace, listeners, appScope, entityId,
+// version, false);
+//
+// final MutationBatch newBatch = mock( MutationBatch.class );
+//
+//
+// // set up returning a mutator
+// when( uvss.delete( same( appScope ), any( UniqueValue.class ) ) ).thenReturn( newBatch );
+//
+// //return a new batch when it's called
+// when( less.delete( same( appScope ), same( entityId ), any( UUID.class ) ) ).thenReturn( newBatch );
+//
+//
+// //start the task
+// ListenableFuture<Void> future = taskExecutor.submit( cleanupTask );
+//
+// /**
+// * While we're not done, release latches every 200 ms
+// */
+// while ( !future.isDone() ) {
+// Thread.sleep( 200 );
+// waitSemaphore.release( listenerCount );
+// }
+//
+// //wait for the task
+// future.get();
+//
+//
+// //get the second field, this should be deleted
+// final UniqueValue oldUniqueField = uniqueValueEntryMock.getEntryAtIndex( 1 );
+//
+// final MvccLogEntry expectedDeletedEntry = logEntryMock.getEntryAtIndex( 1 );
+//
+//
+// //verify delete was invoked
+// verify( uvss ).delete( same( appScope ), same( oldUniqueField ) );
+//
+// //verify the delete was invoked
+// verify( less ).delete( same( appScope ), same( entityId ), same( expectedDeletedEntry.getVersion() ) );
+//
+// // verify it was run
+// verify( entityBatch ).execute();
+//
+//
+// //the latch was executed
+// latch.await();
+//
+// //we deleted the version
+// //verify we deleted everything
+// //verify delete was invoked
+// verify( uvss ).delete( same( appScope ), same( oldUniqueField ) );
+//
+// //verify the delete was invoked
+// verify( less ).delete( same( appScope ), same( entityId ), same( expectedDeletedEntry.getVersion() ) );
+//
+// // verify it was run
+// verify( entityBatch ).execute();
+//
+// //the latch was executed
+// latch.await();
+//
+//
+// //the latch was executed
+// latch.await();
+// }
+//
+//
+// /**
+// * Tests that our task will run in the caller if there's no threads, ensures that the task runs
+// */
+// @Test( timeout = 10000 )
+// public void singleListenerSingleVersionRejected()
+// throws ExecutionException, InterruptedException, ConnectionException {
+//
+//
+//
+// //create a latch for the event listener, and add it to the list of events
+// final int sizeToReturn = 1;
+//
+// final CountDownLatch latch = new CountDownLatch( sizeToReturn );
+//
+// final EntityVersionDeletedTest eventListener = new EntityVersionDeletedTest( latch );
+//
+// final Set<EntityVersionDeleted> listeners = new HashSet<>();
+//
+// listeners.add( eventListener );
+//
+//
+// final SerializationFig serializationFig = mock( SerializationFig.class );
+//
+// when( serializationFig.getBufferSize() ).thenReturn( 10 );
+//
+// final MvccLogEntrySerializationStrategy less = mock( MvccLogEntrySerializationStrategy.class );
+//
+// final UniqueValueSerializationStrategy uvss = mock( UniqueValueSerializationStrategy.class );
+//
+// final Keyspace keyspace = mock( Keyspace.class );
+//
+// final MutationBatch entityBatch = mock( MutationBatch.class );
+//
+// when( keyspace.prepareMutationBatch() ).thenReturn(
+// mock( MutationBatch.class ) ) // don't care what happens to this one
+// .thenReturn( entityBatch );
+//
+//
+// final Id applicationId = new SimpleId( "application" );
+//
+// final ApplicationScope appScope = new ApplicationScopeImpl( applicationId );
+//
+// final Id entityId = new SimpleId( "user" );
+//
+//
+// final List<UUID> versions = VersionGenerator.generateVersions( 2 );
+//
+//
+// // mock up a single log entry for our first test
+// final LogEntryMock logEntryMock = LogEntryMock.createLogEntryMock( less, appScope, entityId, versions );
+//
+//
+// //get the version we're keeping, it's first in our list
+// final UUID version = logEntryMock.getEntryAtIndex( 0 ).getVersion();
+//
+// //mock up unique version output
+// final UniqueValueEntryMock uniqueValueEntryMock =
+// UniqueValueEntryMock.createUniqueMock( uvss, appScope, entityId, versions );
+//
+//
+// EntityVersionCleanupTask cleanupTask =
+// new EntityVersionCleanupTask( serializationFig, less, uvss, keyspace, listeners, appScope, entityId,
+// version, false );
+//
+// final MutationBatch newBatch = mock( MutationBatch.class );
+//
+//
+// // set up returning a mutator
+// when( uvss.delete( same( appScope ), any( UniqueValue.class ) ) ).thenReturn( newBatch );
+//
+// //return a new batch when it's called
+// when( less.delete( same( appScope ), same( entityId ), any( UUID.class ) ) ).thenReturn( newBatch );
+//
+//
+// cleanupTask.rejected();
+//
+//
+// //get the second field, this should be deleted
+// final UniqueValue oldUniqueField = uniqueValueEntryMock.getEntryAtIndex( 1 );
+//
+// final MvccLogEntry expectedDeletedEntry = logEntryMock.getEntryAtIndex( 1 );
+//
+//
+// //verify delete was invoked
+// verify( uvss ).delete( same( appScope ), same( oldUniqueField ) );
+//
+// //verify the delete was invoked
+// verify( less ).delete( same( appScope ), same( entityId ), same( expectedDeletedEntry.getVersion() ) );
+//
+// // verify it was run
+// verify( entityBatch ).execute();
+//
+//
+// //the latch was executed
+// latch.await();
+// }
+//
+//
+// private static class EntityVersionDeletedTest implements EntityVersionDeleted {
+// final CountDownLatch invocationLatch;
+//
+//
+// private EntityVersionDeletedTest( final CountDownLatch invocationLatch ) {
+// this.invocationLatch = invocationLatch;
+// }
+//
+//
+// @Override
+// public void versionDeleted( final ApplicationScope scope, final Id entityId,
+// final List<MvccLogEntry> entityVersion ) {
+// invocationLatch.countDown();
+// }
+// }
+//
+//
+// private static class SlowListener extends EntityVersionDeletedTest {
+// final Semaphore blockLatch;
+//
+//
+// private SlowListener( final CountDownLatch invocationLatch, final Semaphore blockLatch ) {
+// super( invocationLatch );
+// this.blockLatch = blockLatch;
+// }
+//
+//
+// @Override
+// public void versionDeleted( final ApplicationScope scope, final Id entityId,
+// final List<MvccLogEntry> entityVersion ) {
+//
+// //wait for unblock to happen before counting down invocation latches
+// try {
+// blockLatch.acquire();
+// }
+// catch ( InterruptedException e ) {
+// throw new RuntimeException( e );
+// }
+// super.versionDeleted( scope, entityId, entityVersion );
+// }
+// }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3e2afe23/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/VersionCompactTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/VersionCompactTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/VersionCompactTest.java
new file mode 100644
index 0000000..8ffba71
--- /dev/null
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/VersionCompactTest.java
@@ -0,0 +1,238 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence.collection.mvcc.stage.delete;
+
+
+import java.util.Iterator;
+import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutionException;
+
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
+import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.entity.Id;
+import org.apache.usergrid.persistence.model.entity.SimpleId;
+
+import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
+
+import static org.junit.Assert.*;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+
+public class VersionCompactTest {
+
+// private static final TaskExecutor taskExecutor = new NamedTaskExecutorImpl( "test", 4, 0 );
+//
+// @AfterClass
+// public static void shutdown() {
+// taskExecutor.shutdown();
+// }
+//
+//
+// @Test(timeout=10000)
+// public void noListener()
+// throws ExecutionException, InterruptedException, ConnectionException {
+//
+// // create a latch for the event listener, and add it to the list of events
+//
+// final Set<EntityVersionCreated> listeners = mock( Set.class );
+//
+// when ( listeners.size()).thenReturn( 0 );
+//
+// final Id applicationId = new SimpleId( "application" );
+//
+// final ApplicationScope appScope = new ApplicationScopeImpl(applicationId);
+//
+// final Id entityId = new SimpleId( "user" );
+// final Entity entity = new Entity( entityId );
+//
+// // start the task
+//
+// EntityVersionCreatedTask entityVersionCreatedTask =
+// new EntityVersionCreatedTask( appScope, listeners, entity);
+//
+// try {
+// entityVersionCreatedTask.call();
+// }catch(Exception e){
+// Assert.fail( e.getMessage() );
+// }
+//
+//
+// // wait for the task
+// // future.get();
+//
+// //mocked listener makes sure that the task is called
+// verify( listeners ).size();
+//
+// }
+// @Test(timeout=10000)
+// public void oneListener()
+// throws ExecutionException, InterruptedException, ConnectionException {
+//
+// // create a latch for the event listener, and add it to the list of events
+//
+// final int sizeToReturn = 1;
+//
+// final CountDownLatch latch = new CountDownLatch( sizeToReturn );
+//
+// final EntityVersionCreatedTest eventListener = new EntityVersionCreatedTest(latch);
+//
+// final Set<EntityVersionCreated> listeners = mock( Set.class );
+// final Iterator<EntityVersionCreated> helper = mock(Iterator.class);
+//
+// when ( listeners.size()).thenReturn( 1 );
+// when ( listeners.iterator()).thenReturn( helper );
+// when ( helper.next() ).thenReturn( eventListener );
+//
+// final Id applicationId = new SimpleId( "application" );
+//
+// final ApplicationScope appScope = new ApplicationScopeImpl(applicationId);
+//
+// final Id entityId = new SimpleId( "user" );
+// final Entity entity = new Entity( entityId );
+//
+// // start the task
+//
+// EntityVersionCreatedTask entityVersionCreatedTask =
+// new EntityVersionCreatedTask( appScope, listeners, entity);
+//
+// try {
+// entityVersionCreatedTask.call();
+// }catch(Exception e){
+//
+// Assert.fail(e.getMessage());
+// }
+// //mocked listener makes sure that the task is called
+// verify( listeners ).size();
+// verify( listeners ).iterator();
+// verify( helper ).next();
+//
+// }
+//
+// @Test(timeout=10000)
+// public void multipleListener()
+// throws ExecutionException, InterruptedException, ConnectionException {
+//
+// final int sizeToReturn = 3;
+//
+// final Set<EntityVersionCreated> listeners = mock( Set.class );
+// final Iterator<EntityVersionCreated> helper = mock(Iterator.class);
+//
+// when ( listeners.size()).thenReturn( 3 );
+// when ( listeners.iterator()).thenReturn( helper );
+//
+// final Id applicationId = new SimpleId( "application" );
+//
+// final ApplicationScope appScope = new ApplicationScopeImpl(applicationId);
+//
+// final Id entityId = new SimpleId( "user" );
+// final Entity entity = new Entity( entityId );
+//
+// // start the task
+//
+// EntityVersionCreatedTask entityVersionCreatedTask =
+// new EntityVersionCreatedTask( appScope, listeners, entity);
+//
+// final CountDownLatch latch = new CountDownLatch( sizeToReturn );
+//
+// final EntityVersionCreatedTest listener1 = new EntityVersionCreatedTest(latch);
+// final EntityVersionCreatedTest listener2 = new EntityVersionCreatedTest(latch);
+// final EntityVersionCreatedTest listener3 = new EntityVersionCreatedTest(latch);
+//
+// when ( helper.next() ).thenReturn( listener1,listener2,listener3);
+//
+// try {
+// entityVersionCreatedTask.call();
+// }catch(Exception e){
+// ;
+// }
+// //ListenableFuture<Void> future = taskExecutor.submit( entityVersionCreatedTask );
+//
+// //wait for the task
+// //intentionally fails due to difficulty mocking observable
+//
+// //mocked listener makes sure that the task is called
+// verify( listeners ).size();
+// //verifies that the observable made listener iterate.
+// verify( listeners ).iterator();
+// }
+//
+// @Test(timeout=10000)
+// public void oneListenerRejected()
+// throws ExecutionException, InterruptedException, ConnectionException {
+//
+// // create a latch for the event listener, and add it to the list of events
+//
+// final TaskExecutor taskExecutor = new NamedTaskExecutorImpl( "test", 0, 0 );
+//
+// final int sizeToReturn = 1;
+//
+// final CountDownLatch latch = new CountDownLatch( sizeToReturn );
+//
+// final EntityVersionCreatedTest eventListener = new EntityVersionCreatedTest(latch);
+//
+// final Set<EntityVersionCreated> listeners = mock( Set.class );
+// final Iterator<EntityVersionCreated> helper = mock(Iterator.class);
+//
+// when ( listeners.size()).thenReturn( 1 );
+// when ( listeners.iterator()).thenReturn( helper );
+// when ( helper.next() ).thenReturn( eventListener );
+//
+// final Id applicationId = new SimpleId( "application" );
+//
+// final ApplicationScope appScope = new ApplicationScopeImpl(applicationId);
+//
+// final Id entityId = new SimpleId( "user" );
+// final Entity entity = new Entity( entityId );
+//
+// // start the task
+//
+// EntityVersionCreatedTask entityVersionCreatedTask =
+// new EntityVersionCreatedTask( appScope, listeners, entity);
+//
+// entityVersionCreatedTask.rejected();
+//
+// //mocked listener makes sure that the task is called
+// verify( listeners ).size();
+// verify( listeners ).iterator();
+// verify( helper ).next();
+//
+// }
+//
+// private static class EntityVersionCreatedTest implements EntityVersionCreated {
+// final CountDownLatch invocationLatch;
+//
+// private EntityVersionCreatedTest( final CountDownLatch invocationLatch) {
+// this.invocationLatch = invocationLatch;
+// }
+//
+// @Override
+// public void versionCreated( final ApplicationScope scope, final Entity entity ) {
+// invocationLatch.countDown();
+// }
+// }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3e2afe23/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/LogEntryIteratorTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/LogEntryIteratorTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/LogEntryIteratorTest.java
deleted file mode 100644
index 720e602..0000000
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/LogEntryIteratorTest.java
+++ /dev/null
@@ -1,132 +0,0 @@
-package org.apache.usergrid.persistence.collection.serialization.impl;
-
-
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.UUID;
-
-import org.junit.Test;
-
-import org.apache.usergrid.persistence.collection.MvccLogEntry;
-import org.apache.usergrid.persistence.collection.serialization.MvccLogEntrySerializationStrategy;
-import org.apache.usergrid.persistence.collection.util.LogEntryMock;
-import org.apache.usergrid.persistence.collection.util.VersionGenerator;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
-import org.apache.usergrid.persistence.model.entity.Id;
-import org.apache.usergrid.persistence.model.entity.SimpleId;
-import org.apache.usergrid.persistence.model.util.UUIDGenerator;
-
-import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.mockito.Matchers.same;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-
-/**
- * Tests iterator paging
- */
-public class LogEntryIteratorTest {
-
-
- @Test
- public void empty() throws ConnectionException {
-
- final MvccLogEntrySerializationStrategy logEntrySerializationStrategy =
- mock( MvccLogEntrySerializationStrategy.class );
-
- final ApplicationScope scope =
- new ApplicationScopeImpl( new SimpleId( "application" ));
-
- final Id entityId = new SimpleId( "entity" );
-
- final int pageSize = 100;
-
-
- //set the start version, it should be discarded
- UUID start = UUIDGenerator.newTimeUUID();
-
- when( logEntrySerializationStrategy.load( same( scope ), same( entityId ), same( start ), same( pageSize ) ) )
- .thenReturn( new ArrayList<MvccLogEntry>() );
-
-
- //now iterate we should get everything
- LogEntryIterator itr = new LogEntryIterator( logEntrySerializationStrategy, scope, entityId, start, pageSize );
-
-
- assertFalse( itr.hasNext() );
- }
-
-
- @Test
- public void partialLastPage() throws ConnectionException {
-
-
- final int pageSize = 10;
- final int totalPages = 3;
- final int lastPageSize = pageSize / 2;
-
- //have one half page
-
- pageElements( pageSize, totalPages, lastPageSize );
- }
-
-
- @Test
- public void emptyLastPage() throws ConnectionException {
-
-
- final int pageSize = 10;
- final int totalPages = 3;
- final int lastPageSize = 0;
-
- //have one half page
-
- pageElements( pageSize, totalPages, lastPageSize );
- }
-
-
- public void pageElements( final int pageSize, final int totalPages, final int lastPageSize )
- throws ConnectionException {
-
- final MvccLogEntrySerializationStrategy logEntrySerializationStrategy =
- mock( MvccLogEntrySerializationStrategy.class );
-
- final ApplicationScope scope =
- new ApplicationScopeImpl( new SimpleId( "application" ) );
-
- final Id entityId = new SimpleId( "entity" );
-
-
- //have one half page
- final int toGenerate = pageSize * totalPages + lastPageSize;
-
-
- final LogEntryMock mockResults =
- LogEntryMock.createLogEntryMock( logEntrySerializationStrategy, scope, entityId, VersionGenerator.generateVersions( toGenerate ) );
-
- Iterator<MvccLogEntry> expectedEntries = mockResults.getEntries().iterator();
-
- //this element should be skipped
- UUID start = expectedEntries.next().getVersion();
-
- //now iterate we should get everything
- LogEntryIterator itr = new LogEntryIterator( logEntrySerializationStrategy, scope, entityId, start, pageSize );
-
-
- while ( expectedEntries.hasNext() && itr.hasNext() ) {
- final MvccLogEntry expected = expectedEntries.next();
-
- final MvccLogEntry returned = itr.next();
-
- assertEquals( expected, returned );
- }
-
-
- assertFalse( itr.hasNext() );
- assertFalse( expectedEntries.hasNext() );
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3e2afe23/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MinMaxLogEntryIteratorTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MinMaxLogEntryIteratorTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MinMaxLogEntryIteratorTest.java
new file mode 100644
index 0000000..a7210a5
--- /dev/null
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MinMaxLogEntryIteratorTest.java
@@ -0,0 +1,131 @@
+package org.apache.usergrid.persistence.collection.serialization.impl;
+
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.UUID;
+
+import org.junit.Test;
+
+import org.apache.usergrid.persistence.collection.MvccLogEntry;
+import org.apache.usergrid.persistence.collection.serialization.MvccLogEntrySerializationStrategy;
+import org.apache.usergrid.persistence.collection.util.LogEntryMock;
+import org.apache.usergrid.persistence.collection.util.VersionGenerator;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
+import org.apache.usergrid.persistence.model.entity.Id;
+import org.apache.usergrid.persistence.model.entity.SimpleId;
+import org.apache.usergrid.persistence.model.util.UUIDGenerator;
+
+import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.mockito.Matchers.same;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+
+/**
+ * Tests iterator paging
+ */
+public class MinMaxLogEntryIteratorTest {
+
+
+ @Test
+ public void empty() throws ConnectionException {
+
+ final MvccLogEntrySerializationStrategy logEntrySerializationStrategy =
+ mock( MvccLogEntrySerializationStrategy.class );
+
+ final ApplicationScope scope =
+ new ApplicationScopeImpl( new SimpleId( "application" ));
+
+ final Id entityId = new SimpleId( "entity" );
+
+ final int pageSize = 100;
+
+
+ //set the start version, it should be discarded
+ UUID start = UUIDGenerator.newTimeUUID();
+
+ when( logEntrySerializationStrategy.load( same( scope ), same( entityId ), same( start ), same( pageSize ) ) )
+ .thenReturn( new ArrayList<MvccLogEntry>() );
+
+
+ //now iterate we should get everything
+ MinMaxLogEntryIterator
+ itr = new MinMaxLogEntryIterator( logEntrySerializationStrategy, scope, entityId, pageSize );
+
+
+ assertFalse( itr.hasNext() );
+ }
+
+
+ @Test
+ public void partialLastPage() throws ConnectionException {
+
+
+ final int pageSize = 10;
+ final int totalPages = 3;
+ final int lastPageSize = pageSize / 2;
+
+ //have one half page
+
+ pageElements( pageSize, totalPages, lastPageSize );
+ }
+
+
+ @Test
+ public void emptyLastPage() throws ConnectionException {
+
+
+ final int pageSize = 10;
+ final int totalPages = 3;
+ final int lastPageSize = 0;
+
+ //have one half page
+
+ pageElements( pageSize, totalPages, lastPageSize );
+ }
+
+
+ public void pageElements( final int pageSize, final int totalPages, final int lastPageSize )
+ throws ConnectionException {
+
+ final MvccLogEntrySerializationStrategy logEntrySerializationStrategy =
+ mock( MvccLogEntrySerializationStrategy.class );
+
+ final ApplicationScope scope =
+ new ApplicationScopeImpl( new SimpleId( "application" ) );
+
+ final Id entityId = new SimpleId( "entity" );
+
+
+ //have one half page
+ final int toGenerate = pageSize * totalPages + lastPageSize;
+
+
+ final LogEntryMock mockResults =
+ LogEntryMock.createLogEntryMock( logEntrySerializationStrategy, scope, entityId, VersionGenerator.generateVersions( toGenerate ) );
+
+ Iterator<MvccLogEntry> expectedEntries = mockResults.getEntries().iterator();
+
+ //now iterate we should get everything
+ MinMaxLogEntryIterator
+ itr = new MinMaxLogEntryIterator( logEntrySerializationStrategy, scope, entityId, pageSize );
+
+
+ while ( expectedEntries.hasNext() && itr.hasNext() ) {
+ final MvccLogEntry expected = expectedEntries.next();
+
+ final MvccLogEntry returned = itr.next();
+
+ assertEquals( expected, returned );
+ }
+
+
+ assertFalse( itr.hasNext() );
+ assertFalse( expectedEntries.hasNext() );
+ }
+}
[4/9] incubator-usergrid git commit: First pass at refactoring mark +
sweep
Posted by to...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3e2afe23/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/UniqueCleanup.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/UniqueCleanup.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/UniqueCleanup.java
new file mode 100644
index 0000000..0034f03
--- /dev/null
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/UniqueCleanup.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence.collection.mvcc.stage.delete;
+
+
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.UUID;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.usergrid.persistence.collection.MvccEntity;
+import org.apache.usergrid.persistence.collection.mvcc.stage.CollectionIoEvent;
+import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
+import org.apache.usergrid.persistence.collection.serialization.UniqueValue;
+import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
+import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
+import org.apache.usergrid.persistence.core.metrics.ObservableTimer;
+import org.apache.usergrid.persistence.core.rx.ObservableIterator;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.codahale.metrics.Timer;
+import com.fasterxml.uuid.UUIDComparator;
+import com.google.inject.Inject;
+import com.netflix.astyanax.Keyspace;
+import com.netflix.astyanax.MutationBatch;
+import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
+
+import rx.Observable;
+
+
+/**
+ * Runs on an entity that as just be mark committed, and removes all unique values <= this entity
+ */
+public class UniqueCleanup
+ implements Observable.Transformer<CollectionIoEvent<MvccEntity>, CollectionIoEvent<MvccEntity>> {
+
+
+ private static final Logger logger = LoggerFactory.getLogger( UniqueCleanup.class );
+ private final Timer uniqueCleanupTimer;
+
+
+ private final UniqueValueSerializationStrategy uniqueValueSerializationStrategy;
+ private final Keyspace keyspace;
+
+ private final SerializationFig serializationFig;
+
+
+ @Inject
+ public UniqueCleanup( final SerializationFig serializationFig,
+ final UniqueValueSerializationStrategy uniqueValueSerializationStrategy,
+ final Keyspace keyspace, final MetricsFactory metricsFactory ) {
+
+ this.serializationFig = serializationFig;
+ this.uniqueValueSerializationStrategy = uniqueValueSerializationStrategy;
+ this.keyspace = keyspace;
+ this.uniqueCleanupTimer = metricsFactory.getTimer( UniqueCleanup.class, "uniquecleanup" );
+ }
+
+
+ @Override
+ public Observable<CollectionIoEvent<MvccEntity>> call(
+ final Observable<CollectionIoEvent<MvccEntity>> collectionIoEventObservable ) {
+
+ final Observable<CollectionIoEvent<MvccEntity>> outputObservable =
+ collectionIoEventObservable.doOnNext( mvccEntityCollectionIoEvent -> {
+
+ final Id entityId = mvccEntityCollectionIoEvent.getEvent().getId();
+ final ApplicationScope applicationScope = mvccEntityCollectionIoEvent.getEntityCollection();
+ final UUID entityVersion = mvccEntityCollectionIoEvent.getEvent().getVersion();
+
+ //TODO Refactor this logic into a a class that can be invoked from anywhere
+ //iterate all unique values
+ final Observable<CollectionIoEvent<MvccEntity>> uniqueValueCleanup =
+ Observable.create( new ObservableIterator<UniqueValue>( "Unique value load" ) {
+ @Override
+ protected Iterator<UniqueValue> getIterator() {
+ return uniqueValueSerializationStrategy.getAllUniqueFields( applicationScope, entityId );
+ }
+ } )
+
+ //skip versions > the specified version
+ .skipWhile( uniqueValue -> {
+
+ final UUID uniqueValueVersion = uniqueValue.getEntityVersion();
+
+ return UUIDComparator.staticCompare( uniqueValueVersion, entityVersion ) > 0;
+ } )
+
+ //buffer our buffer size, then roll them all up in a single batch mutation
+ .buffer( serializationFig.getBufferSize() )
+
+ //roll them up
+ .doOnNext( uniqueValues -> {
+ final MutationBatch uniqueCleanupBatch = keyspace.prepareMutationBatch();
+
+
+ for ( UniqueValue value : uniqueValues ) {
+ uniqueCleanupBatch
+ .mergeShallow( uniqueValueSerializationStrategy.delete( applicationScope, value ) );
+ }
+
+ try {
+ uniqueCleanupBatch.execute();
+ }
+ catch ( ConnectionException e ) {
+ throw new RuntimeException( "Unable to execute batch mutation", e );
+ }
+ } ).lastOrDefault( Collections.emptyList() ).map( list -> mvccEntityCollectionIoEvent );
+ } );
+
+ return ObservableTimer.time( outputObservable, uniqueCleanupTimer );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3e2afe23/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/VersionCompact.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/VersionCompact.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/VersionCompact.java
new file mode 100644
index 0000000..424ec86
--- /dev/null
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/VersionCompact.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence.collection.mvcc.stage.delete;
+
+
+import java.util.UUID;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.usergrid.persistence.collection.MvccLogEntry;
+import org.apache.usergrid.persistence.collection.mvcc.stage.CollectionIoEvent;
+import org.apache.usergrid.persistence.collection.serialization.MvccEntitySerializationStrategy;
+import org.apache.usergrid.persistence.collection.serialization.MvccLogEntrySerializationStrategy;
+import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
+import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
+import org.apache.usergrid.persistence.core.metrics.ObservableTimer;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.codahale.metrics.Timer;
+import com.google.inject.Inject;
+import com.netflix.astyanax.Keyspace;
+import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
+
+import rx.Observable;
+
+
+/**
+ * Compact all versions on the input observable by removing them from the log, and from the
+ * versions
+ */
+public class VersionCompact
+ implements Observable.Transformer<CollectionIoEvent<MvccLogEntry>, CollectionIoEvent<MvccLogEntry>> {
+
+ private static final Logger logger = LoggerFactory.getLogger( VersionCompact.class );
+
+ private final MvccLogEntrySerializationStrategy logEntrySerializationStrategy;
+ private final MvccEntitySerializationStrategy mvccEntitySerializationStrategy;
+ private final SerializationFig serializationFig;
+ private final Keyspace keyspace;
+ private final Timer compactTimer;
+
+
+ @Inject
+ public VersionCompact( final MvccLogEntrySerializationStrategy logEntrySerializationStrategy,
+ final SerializationFig serializationFig, final Keyspace keyspace,
+ final MetricsFactory metricsFactory,
+ final MvccEntitySerializationStrategy mvccEntitySerializationStrategy ) {
+ this.logEntrySerializationStrategy = logEntrySerializationStrategy;
+ this.serializationFig = serializationFig;
+ this.keyspace = keyspace;
+ this.mvccEntitySerializationStrategy = mvccEntitySerializationStrategy;
+ this.compactTimer = metricsFactory.getTimer( VersionCompact.class, "compact" );
+ }
+
+
+ @Override
+ public Observable<CollectionIoEvent<MvccLogEntry>> call(
+ final Observable<CollectionIoEvent<MvccLogEntry>> collectionIoEventObservable ) {
+
+
+ final Observable<CollectionIoEvent<MvccLogEntry>> entryBuffer =
+ collectionIoEventObservable.buffer( serializationFig.getBufferSize() ).flatMap(
+ buffer -> Observable.from( buffer ).collect( () -> keyspace.prepareMutationBatch(),
+ ( ( mutationBatch, mvccLogEntryCollectionIoEvent ) -> {
+
+ final ApplicationScope scope = mvccLogEntryCollectionIoEvent.getEntityCollection();
+ final MvccLogEntry mvccLogEntry = mvccLogEntryCollectionIoEvent.getEvent();
+ final Id entityId = mvccLogEntry.getEntityId();
+ final UUID version = mvccLogEntry.getVersion();
+
+ if ( logger.isDebugEnabled() ) {
+ logger.debug(
+ "Deleting log entry and version data for entity id {} and version {} in app scope {}",
+ new Object[] { entityId, version, scope } );
+ }
+
+
+ //delete from our log
+ mutationBatch.mergeShallow( logEntrySerializationStrategy.delete( scope, entityId, version ) );
+
+ //merge our entity delete in
+ mutationBatch
+ .mergeShallow( mvccEntitySerializationStrategy.delete( scope, entityId, version ) );
+
+
+
+ } ) )
+ //delete from the entities
+ .doOnNext( mutationBatch -> {
+ try {
+ mutationBatch.execute();
+ }
+ catch ( ConnectionException e ) {
+ throw new RuntimeException( "Unable to perform batch mutation" );
+ }
+ } ).flatMap( batches -> Observable.from( buffer ) ) );
+
+
+ return ObservableTimer.time( entryBuffer, compactTimer );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3e2afe23/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/MvccLogEntrySerializationStrategy.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/MvccLogEntrySerializationStrategy.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/MvccLogEntrySerializationStrategy.java
index 92669a7..5e249ae 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/MvccLogEntrySerializationStrategy.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/MvccLogEntrySerializationStrategy.java
@@ -71,6 +71,20 @@ public interface MvccLogEntrySerializationStrategy extends Migration, VersionedD
*/
List<MvccLogEntry> load( ApplicationScope applicationScope, Id entityId, UUID version, int maxSize );
+
+
+ /**
+ * Load a list, from lowest to highest of the stage with versions <= version up to maxSize elements
+ *
+ * @param applicationScope The applicationScope to load the entity from
+ * @param entityId The entity id to load
+ * @param minVersion The min version to seek from. Null is allowed
+ * @param maxSize The maximum size to return. If you receive this size, there may be more versions to load.
+ *
+ * @return A list of entities up to max size ordered from max(UUID)=> min(UUID)
+ */
+ List<MvccLogEntry> loadReversed( ApplicationScope applicationScope, Id entityId, UUID minVersion, int maxSize );
+
/**
* MarkCommit the stage from the applicationScope with the given entityId and version
*
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3e2afe23/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/SerializationFig.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/SerializationFig.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/SerializationFig.java
index 381a24e..6591781 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/SerializationFig.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/SerializationFig.java
@@ -24,42 +24,18 @@ public interface SerializationFig extends GuicyFig {
@Default("5")
int getTimeout();
- /**
- * Number of history items to return for delete.
- *
- * @return Timeout in seconds.
- */
- @Key("collection.delete.history.size")
- @Default("100")
- int getHistorySize();
/**
* Number of items to buffer.
*
- * @return Timeout in seconds.
+ * @return Number of items to buffer in memory
*/
- @Key("collection.buffer.size")
- @Default("10")
+ @Key("buffer.size")
+ @Default("100")
int getBufferSize();
/**
- * The size of threads to have in the task pool
- */
- @Key( "collection.task.pool.threadsize" )
- @Default( "20" )
- int getTaskPoolThreadSize();
-
-
-
- /**
- * The size of threads to have in the task pool
- */
- @Key( "collection.task.pool.queuesize" )
- @Default( "20" )
- int getTaskPoolQueueSize();
-
- /**
* The maximum amount of entities we can load in a single request
* TODO, change this and move it into a common setting that both query and collection share
*/
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3e2afe23/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/LogEntryIterator.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/LogEntryIterator.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/LogEntryIterator.java
deleted file mode 100644
index e5c2896..0000000
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/LogEntryIterator.java
+++ /dev/null
@@ -1,114 +0,0 @@
-package org.apache.usergrid.persistence.collection.serialization.impl;
-
-
-import java.util.Iterator;
-import java.util.List;
-import java.util.NoSuchElementException;
-import java.util.UUID;
-
-import org.apache.usergrid.persistence.collection.MvccLogEntry;
-import org.apache.usergrid.persistence.collection.serialization.MvccLogEntrySerializationStrategy;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.model.entity.Id;
-
-import com.google.common.base.Preconditions;
-import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
-
-
-/**
- * Iterator that will iterate all versions of the entity from the log from < the specified maxVersion
- */
-public class LogEntryIterator implements Iterator<MvccLogEntry> {
-
-
- private final MvccLogEntrySerializationStrategy logEntrySerializationStrategy;
- private final ApplicationScope scope;
- private final Id entityId;
- private final int pageSize;
-
-
- private Iterator<MvccLogEntry> elementItr;
- private UUID nextStart;
-
-
- /**
- * @param logEntrySerializationStrategy The serialization strategy to get the log entries
- * @param scope The scope of the entity
- * @param entityId The id of the entity
- * @param maxVersion The max version of the entity. Iterator will iterate from max to min starting with the version
- * < max
- * @param pageSize The fetch size to get when querying the serialization strategy
- */
- public LogEntryIterator( final MvccLogEntrySerializationStrategy logEntrySerializationStrategy,
- final ApplicationScope scope, final Id entityId, final UUID maxVersion,
- final int pageSize ) {
-
- Preconditions.checkArgument( pageSize > 0, "pageSize must be > 0" );
-
- this.logEntrySerializationStrategy = logEntrySerializationStrategy;
- this.scope = scope;
- this.entityId = entityId;
- this.nextStart = maxVersion;
- this.pageSize = pageSize;
- }
-
-
- @Override
- public boolean hasNext() {
- if ( elementItr == null || !elementItr.hasNext() && nextStart != null ) {
- try {
- advance();
- }
- catch ( ConnectionException e ) {
- throw new RuntimeException( "Unable to query cassandra", e );
- }
- }
-
- return elementItr.hasNext();
- }
-
-
- @Override
- public MvccLogEntry next() {
- if ( !hasNext() ) {
- throw new NoSuchElementException( "No more elements exist" );
- }
-
- return elementItr.next();
- }
-
-
- @Override
- public void remove() {
- throw new UnsupportedOperationException( "Remove is unsupported" );
- }
-
-
- /**
- * Advance our iterator
- */
- public void advance() throws ConnectionException {
-
- final int requestedSize = pageSize + 1;
-
- //loop through even entry that's < this one and remove it
- List<MvccLogEntry> results = logEntrySerializationStrategy.load( scope, entityId, nextStart, requestedSize );
-
- //we always remove the first version if it's equal since it's returned
- if ( results.size() > 0 && results.get( 0 ).getVersion().equals( nextStart ) ) {
- results.remove( 0 );
- }
-
-
- //we have results, set our next start
- if ( results.size() == pageSize ) {
- nextStart = results.get( results.size() - 1 ).getVersion();
- }
- //nothing left to do
- else {
- nextStart = null;
- }
-
- elementItr = results.iterator();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3e2afe23/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MinMaxLogEntryIterator.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MinMaxLogEntryIterator.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MinMaxLogEntryIterator.java
new file mode 100644
index 0000000..eae8c06
--- /dev/null
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MinMaxLogEntryIterator.java
@@ -0,0 +1,121 @@
+package org.apache.usergrid.persistence.collection.serialization.impl;
+
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.NoSuchElementException;
+import java.util.UUID;
+
+import org.apache.usergrid.persistence.collection.MvccLogEntry;
+import org.apache.usergrid.persistence.collection.serialization.MvccLogEntrySerializationStrategy;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.google.common.base.Preconditions;
+import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
+
+
+/**
+ * Iterator that will iterate all versions of the entity from the log from < the specified maxVersion
+ */
+public class MinMaxLogEntryIterator implements Iterator<MvccLogEntry> {
+
+
+ private final MvccLogEntrySerializationStrategy logEntrySerializationStrategy;
+ private final ApplicationScope scope;
+ private final Id entityId;
+ private final int pageSize;
+
+
+ private Iterator<MvccLogEntry> elementItr;
+ private UUID nextStart;
+
+
+ /**
+ * @param logEntrySerializationStrategy The serialization strategy to get the log entries
+ * @param scope The scope of the entity
+ * @param entityId The id of the entity
+ * @param pageSize The fetch size to get when querying the serialization strategy
+ */
+ public MinMaxLogEntryIterator( final MvccLogEntrySerializationStrategy logEntrySerializationStrategy,
+ final ApplicationScope scope, final Id entityId, final int pageSize ) {
+
+ Preconditions.checkArgument( pageSize > 0, "pageSize must be > 0" );
+
+ this.logEntrySerializationStrategy = logEntrySerializationStrategy;
+ this.scope = scope;
+ this.entityId = entityId;
+ this.pageSize = pageSize;
+ }
+
+
+ @Override
+ public boolean hasNext() {
+ if ( elementItr == null || !elementItr.hasNext() && nextStart != null ) {
+ try {
+ advance();
+ }
+ catch ( ConnectionException e ) {
+ throw new RuntimeException( "Unable to query cassandra", e );
+ }
+ }
+
+ return elementItr.hasNext();
+ }
+
+
+ @Override
+ public MvccLogEntry next() {
+ if ( !hasNext() ) {
+ throw new NoSuchElementException( "No more elements exist" );
+ }
+
+ return elementItr.next();
+ }
+
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException( "Remove is unsupported" );
+ }
+
+
+ /**
+ * Advance our iterator
+ */
+ public void advance() throws ConnectionException {
+
+ final int requestedSize;
+
+ if ( nextStart != null ) {
+ requestedSize = pageSize + 1;
+ }
+ else {
+ requestedSize = pageSize;
+ }
+
+ //loop through even entry that's < this one and remove it
+ List<MvccLogEntry> results = logEntrySerializationStrategy.loadReversed( scope, entityId, nextStart, requestedSize );
+
+ //we always remove the first version if it's equal since it's returned
+ if ( nextStart != null && results.size() > 0 && results.get( 0 ).getVersion().equals( nextStart ) ) {
+ results.remove( 0 );
+ }
+
+
+
+ //we have results, set our next start. If we miss our start version (due to deletion) and we request a +1, we want to ensure we set our next, hence the >=
+ if ( results.size() >= pageSize ) {
+ nextStart = results.get( results.size() - 1 ).getVersion();
+ }
+ //nothing left to do
+ else {
+ nextStart = null;
+ }
+
+
+
+
+ elementItr = results.iterator();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3e2afe23/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccLogEntrySerializationProxyImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccLogEntrySerializationProxyImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccLogEntrySerializationProxyImpl.java
index 81c0248..fdef3c3 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccLogEntrySerializationProxyImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccLogEntrySerializationProxyImpl.java
@@ -111,6 +111,20 @@ public class MvccLogEntrySerializationProxyImpl implements MvccLogEntrySerializa
@Override
+ public List<MvccLogEntry> loadReversed( final ApplicationScope applicationScope, final Id entityId,
+ final UUID minVersion, final int maxSize ) {
+
+ final MigrationRelationship<MvccLogEntrySerializationStrategy> migration = getMigrationRelationShip();
+
+ if ( migration.needsMigration() ) {
+ return migration.from.loadReversed( applicationScope, entityId, minVersion, maxSize );
+ }
+
+ return migration.to.loadReversed( applicationScope, entityId, minVersion, maxSize );
+ }
+
+
+ @Override
public MutationBatch delete( final ApplicationScope applicationScope, final Id entityId, final UUID version ) {
final MigrationRelationship<MvccLogEntrySerializationStrategy> migration = getMigrationRelationShip();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3e2afe23/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccLogEntrySerializationStrategyImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccLogEntrySerializationStrategyImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccLogEntrySerializationStrategyImpl.java
index 76e9dba..0c0d961 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccLogEntrySerializationStrategyImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccLogEntrySerializationStrategyImpl.java
@@ -21,7 +21,6 @@ package org.apache.usergrid.persistence.collection.serialization.impl;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collection;
-import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
@@ -31,11 +30,6 @@ import java.util.UUID;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.cassandra.db.marshal.BytesType;
-import org.apache.cassandra.db.marshal.IntegerType;
-import org.apache.cassandra.db.marshal.ReversedType;
-import org.apache.cassandra.db.marshal.UUIDType;
-
import org.apache.usergrid.persistence.collection.MvccLogEntry;
import org.apache.usergrid.persistence.collection.VersionSet;
import org.apache.usergrid.persistence.collection.exception.CollectionRuntimeException;
@@ -43,10 +37,7 @@ import org.apache.usergrid.persistence.collection.mvcc.entity.Stage;
import org.apache.usergrid.persistence.collection.mvcc.entity.impl.MvccLogEntryImpl;
import org.apache.usergrid.persistence.collection.serialization.MvccLogEntrySerializationStrategy;
import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
-import org.apache.usergrid.persistence.collection.serialization.impl.util.LegacyScopeUtils;
-import org.apache.usergrid.persistence.core.astyanax.IdRowCompositeSerializer;
import org.apache.usergrid.persistence.core.astyanax.MultiTennantColumnFamily;
-import org.apache.usergrid.persistence.core.astyanax.MultiTennantColumnFamilyDefinition;
import org.apache.usergrid.persistence.core.astyanax.ScopedRowKey;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.model.entity.Id;
@@ -98,19 +89,16 @@ public abstract class MvccLogEntrySerializationStrategyImpl<K> implements MvccLo
final UUID colName = entry.getVersion();
final StageStatus stageStatus = new StageStatus( stage, entry.getState() );
- return doWrite( collectionScope, entry.getEntityId(), entry.getVersion(), new RowOp() {
- @Override
- public void doOp( final ColumnListMutation<UUID> colMutation ) {
-
- //Write the stage with a timeout, it's set as transient
- if ( stage.isTransient() ) {
- colMutation.putColumn( colName, stageStatus, SER, fig.getTimeout() );
- return;
- }
+ return doWrite( collectionScope, entry.getEntityId(), entry.getVersion(), colMutation -> {
- //otherwise it's persistent, write it with no expiration
- colMutation.putColumn( colName, stageStatus, SER, null );
+ //Write the stage with a timeout, it's set as transient
+ if ( stage.isTransient() ) {
+ colMutation.putColumn( colName, stageStatus, SER, fig.getTimeout() );
+ return;
}
+
+ //otherwise it's persistent, write it with no expiration
+ colMutation.putColumn( colName, stageStatus, SER, null );
} );
}
@@ -126,12 +114,10 @@ public abstract class MvccLogEntrySerializationStrategyImpl<K> implements MvccLo
//didnt put the max in the error message, I don't want to take the string construction hit every time
Preconditions.checkArgument( entityIds.size() <= fig.getMaxLoadSize(),
- "requested size cannot be over configured maximum" );
+ "requested size cannot be over configured maximum" );
final Id applicationId = collectionScope.getApplication();
- final Id ownerId = applicationId;
-
final List<ScopedRowKey<K>> rowKeys = new ArrayList<>( entityIds.size() );
@@ -155,7 +141,7 @@ public abstract class MvccLogEntrySerializationStrategyImpl<K> implements MvccLo
}
catch ( ConnectionException e ) {
throw new CollectionRuntimeException( null, collectionScope, "An error occurred connecting to cassandra",
- e );
+ e );
}
@@ -181,7 +167,7 @@ public abstract class MvccLogEntrySerializationStrategyImpl<K> implements MvccLo
final StageStatus stageStatus = column.getValue( SER );
final MvccLogEntry logEntry =
- new MvccLogEntryImpl( entityId, version, stageStatus.stage, stageStatus.state );
+ new MvccLogEntryImpl( entityId, version, stageStatus.stage, stageStatus.state );
versionResults.addEntry( logEntry );
@@ -208,13 +194,41 @@ public abstract class MvccLogEntrySerializationStrategyImpl<K> implements MvccLo
final ScopedRowKey<K> rowKey = createKey( applicationId, entityId );
+ columns =
+ keyspace.prepareQuery( CF_ENTITY_LOG ).getKey( rowKey ).withColumnRange( version, null, false, maxSize )
+ .execute().getResult();
+ }
+ catch ( ConnectionException e ) {
+ throw new RuntimeException( "Unable to load log entries", e );
+ }
+
+ return parseResults( columns, entityId );
+ }
+
+
+ @Override
+ public List<MvccLogEntry> loadReversed( final ApplicationScope applicationScope, final Id entityId,
+ final UUID minVersion, final int maxSize ) {
+ ColumnList<UUID> columns;
+ try {
+
+ final Id applicationId = applicationScope.getApplication();
+
+ final ScopedRowKey<K> rowKey = createKey( applicationId, entityId );
+
+
columns = keyspace.prepareQuery( CF_ENTITY_LOG ).getKey( rowKey )
- .withColumnRange( version, null, false, maxSize ).execute().getResult();
+ .withColumnRange( minVersion, null, true, maxSize ).execute().getResult();
}
catch ( ConnectionException e ) {
throw new RuntimeException( "Unable to load log entries", e );
}
+ return parseResults( columns, entityId );
+ }
+
+
+ private List<MvccLogEntry> parseResults( final ColumnList<UUID> columns, final Id entityId ) {
List<MvccLogEntry> results = new ArrayList<MvccLogEntry>( columns.size() );
@@ -236,17 +250,10 @@ public abstract class MvccLogEntrySerializationStrategyImpl<K> implements MvccLo
Preconditions.checkNotNull( entityId, "entityId is required" );
Preconditions.checkNotNull( version, "version context is required" );
- return doWrite( context, entityId, version, new RowOp() {
- @Override
- public void doOp( final ColumnListMutation<UUID> colMutation ) {
- colMutation.deleteColumn( version );
- }
- } );
+ return doWrite( context, entityId, version, colMutation -> colMutation.deleteColumn( version ) );
}
-
-
/**
* Simple callback to perform puts and deletes with a common row setup code
*/
@@ -281,12 +288,14 @@ public abstract class MvccLogEntrySerializationStrategyImpl<K> implements MvccLo
return batch;
}
+
protected abstract MultiTennantColumnFamily<ScopedRowKey<K>, UUID> getColumnFamily();
- protected abstract ScopedRowKey<K> createKey(final Id applicationId, final Id entityId);
+ protected abstract ScopedRowKey<K> createKey( final Id applicationId, final Id entityId );
+
+ protected abstract Id getEntityIdFromKey( final ScopedRowKey<K> key );
- protected abstract Id getEntityIdFromKey(final ScopedRowKey<K> key);
/**
* Internal stage shard
@@ -319,7 +328,7 @@ public abstract class MvccLogEntrySerializationStrategyImpl<K> implements MvccLo
*/
private static class StatusCache {
private Map<Integer, MvccLogEntry.State> values =
- new HashMap<Integer, MvccLogEntry.State>( MvccLogEntry.State.values().length );
+ new HashMap<Integer, MvccLogEntry.State>( MvccLogEntry.State.values().length );
private StatusCache() {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3e2afe23/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/MvccEntityDataMigrationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/MvccEntityDataMigrationImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/MvccEntityDataMigrationImpl.java
index 3168817..e825bbc 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/MvccEntityDataMigrationImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/MvccEntityDataMigrationImpl.java
@@ -31,16 +31,12 @@ import org.slf4j.LoggerFactory;
import org.apache.usergrid.persistence.collection.MvccEntity;
import org.apache.usergrid.persistence.collection.MvccLogEntry;
-import org.apache.usergrid.persistence.collection.impl.EntityVersionCleanupTask;
-import org.apache.usergrid.persistence.collection.impl.EntityVersionTaskFactory;
import org.apache.usergrid.persistence.collection.serialization.MvccEntitySerializationStrategy;
import org.apache.usergrid.persistence.collection.serialization.MvccLogEntrySerializationStrategy;
import org.apache.usergrid.persistence.collection.serialization.UniqueValue;
import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
import org.apache.usergrid.persistence.collection.serialization.impl.MvccEntitySerializationStrategyV3Impl;
-import org.apache.usergrid.persistence.collection.serialization.impl.MvccLogEntrySerializationStrategyV2Impl;
import org.apache.usergrid.persistence.collection.serialization.impl.UniqueValueImpl;
-import org.apache.usergrid.persistence.collection.serialization.impl.UniqueValueSerializationStrategyV2Impl;
import org.apache.usergrid.persistence.core.migration.data.DataMigration;
import org.apache.usergrid.persistence.core.migration.data.DataMigrationException;
import org.apache.usergrid.persistence.core.migration.data.MigrationDataProvider;
@@ -76,7 +72,6 @@ public class MvccEntityDataMigrationImpl implements DataMigration<EntityIdScope>
private final Keyspace keyspace;
private final VersionedMigrationSet<MvccEntitySerializationStrategy> allVersions;
- private final EntityVersionTaskFactory entityVersionCleanupFactory;
private final MvccEntitySerializationStrategyV3Impl mvccEntitySerializationStrategyV3;
private final UniqueValueSerializationStrategy uniqueValueSerializationStrategy;
private final MvccLogEntrySerializationStrategy mvccLogEntrySerializationStrategy;
@@ -85,13 +80,11 @@ public class MvccEntityDataMigrationImpl implements DataMigration<EntityIdScope>
@Inject
public MvccEntityDataMigrationImpl( final Keyspace keyspace,
final VersionedMigrationSet<MvccEntitySerializationStrategy> allVersions,
- final EntityVersionTaskFactory entityVersionCleanupFactory,
final MvccEntitySerializationStrategyV3Impl mvccEntitySerializationStrategyV3,
final UniqueValueSerializationStrategy uniqueValueSerializationStrategy,
final MvccLogEntrySerializationStrategy mvccLogEntrySerializationStrategy ) {
this.keyspace = keyspace;
this.allVersions = allVersions;
- this.entityVersionCleanupFactory = entityVersionCleanupFactory;
this.mvccEntitySerializationStrategyV3 = mvccEntitySerializationStrategyV3;
this.uniqueValueSerializationStrategy = uniqueValueSerializationStrategy;
this.mvccLogEntrySerializationStrategy = mvccLogEntrySerializationStrategy;
@@ -162,7 +155,8 @@ public class MvccEntityDataMigrationImpl implements DataMigration<EntityIdScope>
atomicLong.addAndGet( entities.size() );
- List<EntityVersionCleanupTask> entityVersionCleanupTasks = new ArrayList( entities.size() );
+ final List<Id> toSaveIds = new ArrayList<>( entities.size() );
+
for ( EntityToSaveMessage message : entities ) {
final MutationBatch entityRewrite = migration.to.write( message.scope, message.entity );
@@ -187,6 +181,9 @@ public class MvccEntityDataMigrationImpl implements DataMigration<EntityIdScope>
final UUID version = message.entity.getVersion();
+
+ toSaveIds.add( entityId );
+
// re-write the unique
// values
// but this
@@ -215,7 +212,6 @@ public class MvccEntityDataMigrationImpl implements DataMigration<EntityIdScope>
* Migrate the log entry to the new format
*/
for(final MvccLogEntry entry: logEntries){
-
final MutationBatch mb = mvccLogEntrySerializationStrategy.write( message.scope, entry );
totalBatch.mergeShallow( mb );
@@ -223,24 +219,18 @@ public class MvccEntityDataMigrationImpl implements DataMigration<EntityIdScope>
- //schedule our cleanup task to clean up all the data
- final EntityVersionCleanupTask task = entityVersionCleanupFactory
- .getCleanupTask( message.scope, message.entity.getId(), version, false );
- entityVersionCleanupTasks.add( task );
}
executeBatch( migration.to.getImplementationVersion(), totalBatch, observer, atomicLong );
//now run our cleanup task
- for ( EntityVersionCleanupTask entityVersionCleanupTask : entityVersionCleanupTasks ) {
- try {
- entityVersionCleanupTask.call();
- }
- catch ( Exception e ) {
- LOGGER.error( "Unable to run cleanup task", e );
- }
+ for ( Id updatedId : toSaveIds ) {
+
+
+
+
}
} ).subscribeOn( Schedulers.io() );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3e2afe23/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
index baceeb4..36faf62 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
@@ -91,8 +91,7 @@ public class EntityCollectionManagerIT {
public void write() {
- ApplicationScope context =
- new ApplicationScopeImpl( new SimpleId( "organization" ) );
+ ApplicationScope context = new ApplicationScopeImpl( new SimpleId( "organization" ) );
Entity newEntity = new Entity( new SimpleId( "test" ) );
@@ -113,8 +112,7 @@ public class EntityCollectionManagerIT {
public void writeWithUniqueValues() {
- ApplicationScope context =
- new ApplicationScopeImpl( new SimpleId( "organization" ) );
+ ApplicationScope context = new ApplicationScopeImpl( new SimpleId( "organization" ) );
EntityCollectionManager manager = factory.createCollectionManager( context );
@@ -146,8 +144,7 @@ public class EntityCollectionManagerIT {
public void writeAndLoad() {
- ApplicationScope context =
- new ApplicationScopeImpl( new SimpleId( "organization" ) );
+ ApplicationScope context = new ApplicationScopeImpl( new SimpleId( "organization" ) );
Entity newEntity = new Entity( new SimpleId( "test" ) );
@@ -174,7 +171,7 @@ public class EntityCollectionManagerIT {
public void writeLoadDelete() {
- ApplicationScope context = new ApplicationScopeImpl( new SimpleId( "organization" ) );
+ ApplicationScope context = new ApplicationScopeImpl( new SimpleId( "organization" ) );
Entity newEntity = new Entity( new SimpleId( "test" ) );
EntityCollectionManager manager = factory.createCollectionManager( context );
@@ -185,30 +182,27 @@ public class EntityCollectionManagerIT {
assertNotNull( "Id was assigned", createReturned.getId() );
-
- UUID version = createReturned.getVersion();
-
Observable<Entity> loadObservable = manager.load( createReturned.getId() );
Entity loadReturned = loadObservable.toBlocking().lastOrDefault( null );
assertEquals( "Same value", createReturned, loadReturned );
- manager.delete( createReturned.getId() ).toBlocking().last();
+ manager.mark( createReturned.getId() ).toBlocking().last();
loadObservable = manager.load( createReturned.getId() );
//load may return null, use last or default
loadReturned = loadObservable.toBlocking().lastOrDefault( null );
- assertNull("Entity was deleted", loadReturned);
+ assertNull( "Entity was deleted", loadReturned );
}
@Test
public void writeLoadUpdateLoad() {
- ApplicationScope context = new ApplicationScopeImpl( new SimpleId( "organization" ) );
+ ApplicationScope context = new ApplicationScopeImpl( new SimpleId( "organization" ) );
Entity newEntity = new Entity( new SimpleId( "test" ) );
newEntity.setField( new IntegerField( "counter", 1 ) );
@@ -217,36 +211,36 @@ public class EntityCollectionManagerIT {
Observable<Entity> observable = manager.write( newEntity );
- Entity createReturned = observable.toBlocking().lastOrDefault(null);
+ Entity createReturned = observable.toBlocking().lastOrDefault( null );
assertNotNull( "Id was assigned", createReturned.getId() );
- Observable<Entity> loadObservable = manager.load(createReturned.getId());
+ Observable<Entity> loadObservable = manager.load( createReturned.getId() );
Entity loadReturned = loadObservable.toBlocking().lastOrDefault( null );
assertEquals( "Same value", createReturned, loadReturned );
- assertEquals("Field value correct", createReturned.getField("counter"), loadReturned.getField("counter"));
+ assertEquals( "Field value correct", createReturned.getField( "counter" ), loadReturned.getField( "counter" ) );
//update the field to 2
createReturned.setField( new IntegerField( "counter", 2 ) );
//wait for the write to complete
- manager.write( createReturned ).toBlocking().lastOrDefault(null);
+ manager.write( createReturned ).toBlocking().lastOrDefault( null );
loadObservable = manager.load( createReturned.getId() );
- loadReturned = loadObservable.toBlocking().lastOrDefault(null);
+ loadReturned = loadObservable.toBlocking().lastOrDefault( null );
assertEquals( "Same value", createReturned, loadReturned );
- assertEquals("Field value correct", createReturned.getField("counter"), loadReturned.getField("counter"));
+ assertEquals( "Field value correct", createReturned.getField( "counter" ), loadReturned.getField( "counter" ) );
}
@@ -254,30 +248,30 @@ public class EntityCollectionManagerIT {
public void writeAndLoadScopeClosure() {
- ApplicationScope collectionScope1 = new ApplicationScopeImpl(new SimpleId("organization"));
+ ApplicationScope collectionScope1 = new ApplicationScopeImpl( new SimpleId( "organization" ) );
Entity newEntity = new Entity( new SimpleId( "test" ) );
- EntityCollectionManager manager = factory.createCollectionManager(collectionScope1);
+ EntityCollectionManager manager = factory.createCollectionManager( collectionScope1 );
- Observable<Entity> observable = manager.write(newEntity);
+ Observable<Entity> observable = manager.write( newEntity );
Entity createReturned = observable.toBlocking().lastOrDefault( null );
- assertNotNull("Id was assigned", createReturned.getId());
- assertNotNull("Version was assigned", createReturned.getVersion());
+ assertNotNull( "Id was assigned", createReturned.getId() );
+ assertNotNull( "Version was assigned", createReturned.getVersion() );
Observable<Entity> loadObservable = manager.load( createReturned.getId() );
Entity loadReturned = loadObservable.toBlocking().lastOrDefault( null );
- assertEquals("Same value", createReturned, loadReturned);
+ assertEquals( "Same value", createReturned, loadReturned );
- ApplicationScope collectionScope2 = new ApplicationScopeImpl(new SimpleId("organization"));
+ ApplicationScope collectionScope2 = new ApplicationScopeImpl( new SimpleId( "organization" ) );
//now make sure we can't load it from another scope, using the same org
@@ -286,9 +280,7 @@ public class EntityCollectionManagerIT {
Entity loaded = manager2.load( createReturned.getId() ).toBlocking().lastOrDefault( null );
- assertNull("CollectionScope works correctly", loaded);
-
-
+ assertNull( "CollectionScope works correctly", loaded );
}
@@ -296,34 +288,32 @@ public class EntityCollectionManagerIT {
public void writeAndGetField() {
- ApplicationScope collectionScope1 = new ApplicationScopeImpl(new SimpleId("organization"));
+ ApplicationScope collectionScope1 = new ApplicationScopeImpl( new SimpleId( "organization" ) );
Entity newEntity = new Entity( new SimpleId( "test" ) );
Field field = new StringField( "testField", "unique", true );
- newEntity.setField(field);
+ newEntity.setField( field );
EntityCollectionManager manager = factory.createCollectionManager( collectionScope1 );
- Observable<Entity> observable = manager.write(newEntity);
+ Observable<Entity> observable = manager.write( newEntity );
Entity createReturned = observable.toBlocking().lastOrDefault( null );
assertNotNull( "Id was assigned", createReturned.getId() );
- assertNotNull("Version was assigned", createReturned.getVersion());
+ assertNotNull( "Version was assigned", createReturned.getVersion() );
Id id = manager.getIdField( newEntity.getId().getType(), field ).toBlocking().lastOrDefault( null );
assertNotNull( id );
assertEquals( newEntity.getId(), id );
Field fieldNull = new StringField( "testFieldNotThere", "uniquely", true );
- id = manager.getIdField( newEntity.getId().getType(), fieldNull ).toBlocking().lastOrDefault( null );
+ id = manager.getIdField( newEntity.getId().getType(), fieldNull ).toBlocking().lastOrDefault( null );
assertNull( id );
}
-
-
@Test
public void updateVersioning() {
@@ -331,10 +321,10 @@ public class EntityCollectionManagerIT {
Entity origEntity = new Entity( new SimpleId( "testUpdate" ) );
origEntity.setField( new StringField( "testField", "value" ) );
- ApplicationScope context = new ApplicationScopeImpl(new SimpleId("organization"));
+ ApplicationScope context = new ApplicationScopeImpl( new SimpleId( "organization" ) );
EntityCollectionManager manager = factory.createCollectionManager( context );
- Entity returned = manager.write( origEntity ).toBlocking().lastOrDefault(null);
+ Entity returned = manager.write( origEntity ).toBlocking().lastOrDefault( null );
// note its version
UUID oldVersion = returned.getVersion();
@@ -345,7 +335,7 @@ public class EntityCollectionManagerIT {
// partial update entity but we don't have version number
Entity updateEntity = new Entity( origEntity.getId() );
updateEntity.setField( new StringField( "addedField", "other value" ) );
- manager.write( updateEntity ).toBlocking().lastOrDefault(null);
+ manager.write( updateEntity ).toBlocking().lastOrDefault( null );
// get entity now, it must have a new version
returned = manager.load( origEntity.getId() ).toBlocking().lastOrDefault( null );
@@ -354,14 +344,14 @@ public class EntityCollectionManagerIT {
assertNotNull( "A new version must be assigned", newVersion );
// new Version should be > old version
- assertTrue(UUIDComparator.staticCompare(newVersion, oldVersion) > 0);
+ assertTrue( UUIDComparator.staticCompare( newVersion, oldVersion ) > 0 );
}
@Test
public void writeMultiget() {
- final ApplicationScope context = new ApplicationScopeImpl( new SimpleId( "organization" ) );
+ final ApplicationScope context = new ApplicationScopeImpl( new SimpleId( "organization" ) );
final EntityCollectionManager manager = factory.createCollectionManager( context );
final int multigetSize = serializationFig.getMaxLoadSize();
@@ -381,10 +371,10 @@ public class EntityCollectionManagerIT {
final EntitySet entitySet = manager.load( entityIds ).toBlocking().lastOrDefault( null );
- assertNotNull(entitySet);
+ assertNotNull( entitySet );
assertEquals( multigetSize, entitySet.size() );
- assertFalse(entitySet.isEmpty());
+ assertFalse( entitySet.isEmpty() );
/**
* Validate every element exists
@@ -405,7 +395,7 @@ public class EntityCollectionManagerIT {
@Test
public void writeMultigetRepair() {
- final ApplicationScope context = new ApplicationScopeImpl( new SimpleId( "organization" ) );
+ final ApplicationScope context = new ApplicationScopeImpl( new SimpleId( "organization" ) );
final EntityCollectionManager manager = factory.createCollectionManager( context );
final int multigetSize = serializationFig.getMaxLoadSize();
@@ -444,7 +434,7 @@ public class EntityCollectionManagerIT {
assertEquals( "Same entity returned", expected, returned.getEntity().get() );
- assertTrue((Boolean) returned.getEntity().get().getField("updated").getValue());
+ assertTrue( ( Boolean ) returned.getEntity().get().getField( "updated" ).getValue() );
}
}
@@ -452,7 +442,7 @@ public class EntityCollectionManagerIT {
@Test( expected = IllegalArgumentException.class )
public void readTooLarge() {
- final ApplicationScope context = new ApplicationScopeImpl( new SimpleId( "organization" ) );
+ final ApplicationScope context = new ApplicationScopeImpl( new SimpleId( "organization" ) );
final EntityCollectionManager manager = factory.createCollectionManager( context );
final int multigetSize = serializationFig.getMaxLoadSize() + 1;
@@ -474,7 +464,7 @@ public class EntityCollectionManagerIT {
@Test
public void testGetVersion() {
- ApplicationScope context = new ApplicationScopeImpl( new SimpleId( "organization" ) );
+ ApplicationScope context = new ApplicationScopeImpl( new SimpleId( "organization" ) );
final EntityCollectionManager manager = factory.createCollectionManager( context );
@@ -495,7 +485,7 @@ public class EntityCollectionManagerIT {
VersionSet results =
- manager.getLatestVersion( Arrays.asList( created1.getId(), created2.getId() ) ).toBlocking().last();
+ manager.getLatestVersion( Arrays.asList( created1.getId(), created2.getId() ) ).toBlocking().last();
final MvccLogEntry version1Log = results.getMaxVersion( created1.getId() );
@@ -515,7 +505,7 @@ public class EntityCollectionManagerIT {
@Test
public void testVersionLogWrite() {
- ApplicationScope context = new ApplicationScopeImpl( new SimpleId( "organization" ) );
+ ApplicationScope context = new ApplicationScopeImpl( new SimpleId( "organization" ) );
final EntityCollectionManager manager = factory.createCollectionManager( context );
@@ -529,10 +519,10 @@ public class EntityCollectionManagerIT {
final UUID v1Version = v1Created.getVersion();
- final VersionSet resultsV1 = manager.getLatestVersion(Arrays.asList(v1Created.getId())).toBlocking().last();
+ final VersionSet resultsV1 = manager.getLatestVersion( Arrays.asList( v1Created.getId() ) ).toBlocking().last();
- final MvccLogEntry version1Log = resultsV1.getMaxVersion(v1Created.getId());
+ final MvccLogEntry version1Log = resultsV1.getMaxVersion( v1Created.getId() );
assertEquals( v1Created.getId(), version1Log.getEntityId() );
assertEquals( v1Version, version1Log.getVersion() );
assertEquals( MvccLogEntry.State.COMPLETE, version1Log.getState() );
@@ -543,7 +533,7 @@ public class EntityCollectionManagerIT {
final UUID v2Version = v2Created.getVersion();
- assertTrue("Newer version in v2", UUIDComparator.staticCompare(v2Version, v1Version) > 0);
+ assertTrue( "Newer version in v2", UUIDComparator.staticCompare( v2Version, v1Version ) > 0 );
final VersionSet resultsV2 = manager.getLatestVersion( Arrays.asList( v1Created.getId() ) ).toBlocking().last();
@@ -560,7 +550,7 @@ public class EntityCollectionManagerIT {
@Test
public void testVersionLogUpdate() {
- ApplicationScope context = new ApplicationScopeImpl( new SimpleId( "organization" ) );
+ ApplicationScope context = new ApplicationScopeImpl( new SimpleId( "organization" ) );
final EntityCollectionManager manager = factory.createCollectionManager( context );
@@ -608,7 +598,7 @@ public class EntityCollectionManagerIT {
@Test
public void healthTest() {
- ApplicationScope context = new ApplicationScopeImpl( new SimpleId( "organization" ) );
+ ApplicationScope context = new ApplicationScopeImpl( new SimpleId( "organization" ) );
final EntityCollectionManager manager = factory.createCollectionManager( context );
@@ -632,7 +622,7 @@ public class EntityCollectionManagerIT {
final Entity entity = EntityHelper.generateEntity( setSize );
//now we have one massive, entity, save it and retrieve it.
- ApplicationScope context = new ApplicationScopeImpl( new SimpleId( "organization" ) );
+ ApplicationScope context = new ApplicationScopeImpl( new SimpleId( "organization" ) );
final EntityCollectionManager manager = factory.createCollectionManager( context );
@@ -652,20 +642,21 @@ public class EntityCollectionManagerIT {
SetConfigTestBypass.setValueByPass( serializationFig, "getMaxEntitySize", currentMaxSize + "" );
}
+
@Test
public void invalidNameRepair() throws ConnectionException {
//write an entity with a unique field
- ApplicationScope context = new ApplicationScopeImpl( new SimpleId( "organization" ) );
+ ApplicationScope context = new ApplicationScopeImpl( new SimpleId( "organization" ) );
Entity newEntity = new Entity( new SimpleId( "test" ) );
//if we add a second field we get a second entity that is the exact same. Is this expected?
- final IntegerField expectedInteger = new IntegerField( "count", 5, true );
- // final StringField expectedString = new StringField( "yes", "fred", true );
+ final IntegerField expectedInteger = new IntegerField( "count", 5, true );
+ // final StringField expectedString = new StringField( "yes", "fred", true );
newEntity.setField( expectedInteger );
- // newEntity.setField( expectedString );
+ // newEntity.setField( expectedString );
EntityCollectionManager manager = factory.createCollectionManager( context );
@@ -677,23 +668,26 @@ public class EntityCollectionManagerIT {
assertNotNull( "Id was assigned", createReturned.getId() );
assertNotNull( "Version was assigned", createReturned.getVersion() );
- FieldSet
- fieldResults = manager.getEntitiesFromFields( newEntity.getId().getType(), Arrays.<Field>asList( expectedInteger ) ).toBlocking().last();
+ FieldSet fieldResults =
+ manager.getEntitiesFromFields( newEntity.getId().getType(), Arrays.<Field>asList( expectedInteger ) )
+ .toBlocking().last();
- assertEquals(1,fieldResults.size());
+ assertEquals( 1, fieldResults.size() );
//verify the entity is correct.
- assertEquals( "Same value", createReturned, fieldResults.getEntity( expectedInteger ).getEntity().get()); //loadReturned );
+ assertEquals( "Same value", createReturned,
+ fieldResults.getEntity( expectedInteger ).getEntity().get() ); //loadReturned );
//use the entity serializationStrategy to remove the entity data.
//do a mark as one test, and a delete as another
- entitySerializationStrategy.delete( context,createReturned.getId(),createReturned.getVersion() ).execute();
+ entitySerializationStrategy.delete( context, createReturned.getId(), createReturned.getVersion() ).execute();
//try to load via the unique field, should have triggered repair
- final FieldSet
- results = manager.getEntitiesFromFields( newEntity.getId().getType(), Arrays.<Field>asList( expectedInteger ) ).toBlocking().last();
+ final FieldSet results =
+ manager.getEntitiesFromFields( newEntity.getId().getType(), Arrays.<Field>asList( expectedInteger ) )
+ .toBlocking().last();
//verify no entity returned
@@ -701,37 +695,104 @@ public class EntityCollectionManagerIT {
//user the unique serialization to verify it's been deleted from cassandra
- UniqueValueSet uniqueValues = uniqueValueSerializationStrategy.load( context, newEntity.getId().getType(), createReturned.getFields() );
+ UniqueValueSet uniqueValues =
+ uniqueValueSerializationStrategy.load( context, newEntity.getId().getType(), createReturned.getFields() );
assertFalse( uniqueValues.iterator().hasNext() );
-
}
@Test
public void testGetIdField() throws Exception {
- ApplicationScope context = new ApplicationScopeImpl( new SimpleId( "organization" ) );
+ ApplicationScope context = new ApplicationScopeImpl( new SimpleId( "organization" ) );
EntityCollectionManager manager = factory.createCollectionManager( context );
// create an entity of type "item" with a unique_id field value = 1
Entity entity1 = new Entity( new SimpleId( "item" ) );
- entity1.setField( new StringField( "unique_id", "1", true ));
+ entity1.setField( new StringField( "unique_id", "1", true ) );
manager.write( entity1 ).toBlocking().last();
- final Observable<Id> idObs = manager.getIdField("item", new StringField("unique_id", "1"));
- Id id = idObs.toBlocking().lastOrDefault(null);
- assertEquals(entity1.getId(), id);
+ final Observable<Id> idObs = manager.getIdField( "item", new StringField( "unique_id", "1" ) );
+ Id id = idObs.toBlocking().lastOrDefault( null );
+ assertEquals( entity1.getId(), id );
// create an entity of type "deleted_item" with a unique_id field value = 1
Entity entity2 = new Entity( new SimpleId( "deleted_item" ) );
- entity2.setField( new StringField( "unique_id", "1", true ));
+ entity2.setField( new StringField( "unique_id", "1", true ) );
manager = factory.createCollectionManager( context );
manager.write( entity2 ).toBlocking().last();
- final Observable<Id> id2Obs = manager.getIdField("deleted_item", new StringField("unique_id", "1"));
- Id id2 = id2Obs.toBlocking().lastOrDefault(null);
- assertEquals(entity2.getId(), id2);
+ final Observable<Id> id2Obs = manager.getIdField( "deleted_item", new StringField( "unique_id", "1" ) );
+ Id id2 = id2Obs.toBlocking().lastOrDefault( null );
+ assertEquals( entity2.getId(), id2 );
+ }
+
+
+ @Test
+ public void writeGetVersionsDelete() {
+
+ ApplicationScope context = new ApplicationScopeImpl( new SimpleId( "organization" ) );
+
+ Entity entity = new Entity( new SimpleId( "test" ) );
+ entity.setField( new IntegerField( "counter", 0 ) );
+
+ EntityCollectionManager manager = factory.createCollectionManager( context );
+
+ Entity createReturned = manager.write( entity ).toBlocking().lastOrDefault( null );
+
+ assertNotNull( "Id was assigned", createReturned.getId() );
+
+ final int size = 200;
+
+ final Id entityId = createReturned.getId();
+
+ List<UUID> versions = new ArrayList<>( size );
+ versions.add( entity.getVersion() );
+
+ //write new versions
+ for ( int i = 1; i < size; i++ ) {
+ final Entity newEntity = new Entity( entityId );
+
+ final Entity returnedEntity = manager.write( newEntity ).toBlocking().last();
+
+ versions.add( returnedEntity.getVersion() );
+ }
+
+
+ //now get our values, and load the latest version
+
+ final Entity lastVersion = manager.load( entityId ).toBlocking().last();
+
+ //ensure the latest version is correct
+ assertEquals( versions.get( versions.size() - 1 ), lastVersion.getVersion() );
+
+
+ // now ensure all versions are correct
+ final List<MvccLogEntry> entries = manager.getVersions( entityId ).toList().toBlocking().last();
+
+
+ assertEquals( "Same size expected", versions.size(), entries.size() );
+
+ for ( int i = 0; i < versions.size(); i++ ) {
+ assertEquals( versions.get( i ), entries.get( i ).getVersion() );
+ }
+
+
+ //now get all the log versions, and delete them all we do it in 2+ batches to ensure we clean up as expected
+ manager.getVersions( entityId ).buffer( 100 ).flatMap( bufferList -> manager.delete( bufferList ) )
+ .toBlocking().last();
+
+
+ //now load them, there shouldn't be any versions
+ final List<MvccLogEntry> postDeleteEntries = manager.getVersions( entityId ).toList().toBlocking().last();
+
+ assertEquals( "All log entries should be removed", 0, postDeleteEntries.size() );
+
+ final Entity postDeleteLastVersion = manager.load( entityId ).toBlocking().lastOrDefault( null );
+
+ //ensure the latest version is correct
+ assertNull( "Last version was deleted", postDeleteLastVersion );
}
}
[9/9] incubator-usergrid git commit: Merge branch 'two-dot-o-dev' of
https://git-wip-us.apache.org/repos/asf/incubator-usergrid into USERGRID-614
Posted by to...@apache.org.
Merge branch 'two-dot-o-dev' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into USERGRID-614
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/d0bde188
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/d0bde188
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/d0bde188
Branch: refs/heads/two-dot-o-dev
Commit: d0bde18841d501cc22fbb9977c14cca0b101120c
Parents: 62e650b cf8b551
Author: Todd Nine <tn...@apigee.com>
Authored: Sun May 10 04:59:41 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Sun May 10 04:59:41 2015 -0600
----------------------------------------------------------------------
.../batch/service/SchedulerServiceImpl.java | 2 +-
.../usergrid/corepersistence/CoreModule.java | 2 +
.../corepersistence/CpEntityManager.java | 48 ++---
.../corepersistence/CpEntityManagerFactory.java | 9 +-
.../corepersistence/CpRelationManager.java | 65 +++++--
.../corepersistence/EntityManagerFig.java | 38 ++++
.../asyncevents/InMemoryAsyncEventService.java | 2 +-
.../pipeline/PipelineDiagram.jpg | Bin 0 -> 183035 bytes
.../usergrid/corepersistence/pipeline/README.md | 187 +++++++++++++++++++
.../pipeline/cursor/CursorDiagram.jpg | Bin 0 -> 167390 bytes
.../pipeline/read/ReadDiagram.jpg | Bin 0 -> 818565 bytes
.../pipeline/read/collect/CollectDiagram.jpg | Bin 0 -> 98007 bytes
.../elasticsearch/CandidateEntityFilter.java | 2 +-
.../read/elasticsearch/Elasticsearchdiagram.jpg | Bin 0 -> 316655 bytes
.../pipeline/read/graph/GraphDiagram.jpg | Bin 0 -> 347711 bytes
.../usergrid/persistence/ConnectionRef.java | 4 +-
.../usergrid/persistence/EntityManager.java | 3 +
.../usergrid/persistence/RelationManager.java | 10 +
.../apache/usergrid/persistence/Results.java | 15 +-
.../cassandra/ConnectionRefImpl.java | 24 +--
.../org/apache/usergrid/CoreApplication.java | 2 +-
.../corepersistence/StaleIndexCleanupTest.java | 21 ++-
.../index/AsyncIndexServiceTest.java | 42 ++---
.../corepersistence/index/IndexServiceTest.java | 23 +--
.../java/org/apache/usergrid/mq/MessagesIT.java | 6 +
.../usergrid/persistence/CollectionIT.java | 23 ++-
.../persistence/EntityConnectionsIT.java | 20 +-
.../org/apache/usergrid/persistence/GeoIT.java | 60 +++---
.../persistence/GeoQueryBooleanTest.java | 4 +-
.../usergrid/persistence/PathQueryIT.java | 6 +-
.../PerformanceEntityRebuildIndexTest.java | 11 +-
.../query/IntersectionTransitivePagingIT.java | 3 +-
.../query/IntersectionUnionPagingIT.java | 1 +
.../usergrid-scheduler-test.properties | 2 +
.../index/impl/EsIndexBufferConsumerImpl.java | 2 +-
.../index/impl/IndexRefreshCommandImpl.java | 67 ++++---
.../persistence/index/impl/IndexingUtils.java | 4 +-
.../index/impl/EntityIndexMapUtils.java | 5 +-
.../persistence/index/impl/EntityIndexTest.java | 36 ++--
.../src/test/resources/usergrid-UNIT.properties | 4 +
.../org/apache/usergrid/rest/IndexResource.java | 5 +-
.../rest/test/RefreshIndexResource.java | 2 +-
.../management/export/ExportServiceImpl.java | 2 +-
.../management/importer/ImportServiceImpl.java | 6 +-
.../services/AbstractCollectionService.java | 4 +-
.../usergrid/services/ServiceRequest.java | 2 +-
.../resources/usergrid-custom-test.properties | 3 +
47 files changed, 519 insertions(+), 258 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d0bde188/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d0bde188/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d0bde188/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
[7/9] incubator-usergrid git commit: Fixes runtime bug with cached
entities.
Posted by to...@apache.org.
Fixes runtime bug with cached entities.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/9a89437b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/9a89437b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/9a89437b
Branch: refs/heads/two-dot-o-dev
Commit: 9a89437b18dc3f14b3fcdea569d18e0f9fb64294
Parents: 3e2afe2
Author: Todd Nine <tn...@apigee.com>
Authored: Fri May 8 13:10:00 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Sun May 10 04:59:22 2015 -0600
----------------------------------------------------------------------
.../EntityCollectionManagerFactoryImpl.java | 8 ++--
.../mvcc/stage/delete/VersionCompact.java | 11 ++++--
.../MvccEntitySerializationStrategyV3Impl.java | 39 ++++----------------
.../collection/EntityCollectionManagerIT.java | 3 +-
...MvccEntitySerializationStrategyImplTest.java | 2 +-
5 files changed, 24 insertions(+), 39 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9a89437b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerFactoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerFactoryImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerFactoryImpl.java
index 50a4bfc..6d8717e 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerFactoryImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerFactoryImpl.java
@@ -92,10 +92,12 @@ public class EntityCollectionManagerFactoryImpl implements EntityCollectionManag
metricsFactory, serializationFig,
rxTaskScheduler, scope );
+// TODO temporarily removed If we can avoid this, that would be great
+// final EntityCollectionManager proxy = new CachedEntityCollectionManager(entityCacheFig, target );
+//
+// return proxy;
- final EntityCollectionManager proxy = new CachedEntityCollectionManager(entityCacheFig, target );
-
- return proxy;
+ return target;
}
} );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9a89437b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/VersionCompact.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/VersionCompact.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/VersionCompact.java
index 424ec86..e20ad30 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/VersionCompact.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/VersionCompact.java
@@ -38,6 +38,7 @@ import org.apache.usergrid.persistence.model.entity.Id;
import com.codahale.metrics.Timer;
import com.google.inject.Inject;
import com.netflix.astyanax.Keyspace;
+import com.netflix.astyanax.MutationBatch;
import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
import rx.Observable;
@@ -95,11 +96,15 @@ public class VersionCompact
//delete from our log
- mutationBatch.mergeShallow( logEntrySerializationStrategy.delete( scope, entityId, version ) );
+ final MutationBatch logDelete = logEntrySerializationStrategy.delete( scope, entityId, version );
+
+ mutationBatch.mergeShallow( logDelete );
//merge our entity delete in
- mutationBatch
- .mergeShallow( mvccEntitySerializationStrategy.delete( scope, entityId, version ) );
+
+ final MutationBatch entityDelete = mvccEntitySerializationStrategy.delete( scope, entityId, version );
+
+ mutationBatch.mergeShallow( entityDelete );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9a89437b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3Impl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3Impl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3Impl.java
index be08da8..6418ac7 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3Impl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3Impl.java
@@ -107,13 +107,8 @@ public class MvccEntitySerializationStrategyV3Impl implements MvccEntitySerializ
final Id entityId = entity.getId();
final UUID version = entity.getVersion();
- return doWrite( applicationScope, entityId, version, new RowOp() {
- @Override
- public void doOp( final ColumnListMutation<Boolean> colMutation ) {
- colMutation.putColumn( COL_VALUE,
- entitySerializer.toByteBuffer( new EntityWrapper( entity.getStatus(), entity.getVersion(), entity.getEntity() ) ) );
- }
- } );
+ return doWrite( applicationScope, entityId, version, colMutation -> colMutation.putColumn( COL_VALUE,
+ entitySerializer.toByteBuffer( new EntityWrapper( entity.getStatus(), entity.getVersion(), entity.getEntity() ) ) ) );
}
@@ -135,8 +130,6 @@ public class MvccEntitySerializationStrategyV3Impl implements MvccEntitySerializ
final Id applicationId = applicationScope.getApplication();
- final Id ownerId = applicationId;
-
final List<ScopedRowKey<Id>> rowKeys = new ArrayList<>( entityIds.size() );
@@ -196,11 +189,8 @@ public class MvccEntitySerializationStrategyV3Impl implements MvccEntitySerializ
"An error occurred connecting to cassandra", e );
}
} ).subscribeOn( scheduler );
- }, 10 )
-
- .reduce( new EntitySetImpl( entityIds.size() ), ( entitySet, rows ) -> {
- final Iterator<Row<ScopedRowKey<Id>, Boolean>> latestEntityColumns =
- rows.iterator();
+ }, 10 ).collect( () -> new EntitySetImpl( entityIds.size() ), ( ( entitySet, rows ) -> {
+ final Iterator<Row<ScopedRowKey<Id>, Boolean>> latestEntityColumns = rows.iterator();
while ( latestEntityColumns.hasNext() ) {
final Row<ScopedRowKey<Id>, Boolean> row = latestEntityColumns.next();
@@ -221,10 +211,7 @@ public class MvccEntitySerializationStrategyV3Impl implements MvccEntitySerializ
entitySet.addEntity( parsedEntity );
}
-
-
- return entitySet;
- } ).toBlocking().last();
+ } ) ).toBlocking().last();
@@ -275,13 +262,8 @@ public class MvccEntitySerializationStrategyV3Impl implements MvccEntitySerializ
Preconditions.checkNotNull( version, "version is required" );
- return doWrite( applicationScope, entityId, version, new RowOp() {
- @Override
- public void doOp( final ColumnListMutation<Boolean> colMutation ) {
- colMutation.putColumn( COL_VALUE, entitySerializer.toByteBuffer(
- new EntityWrapper( MvccEntity.Status.COMPLETE, version, Optional.<Entity>absent() ) ) );
- }
- } );
+ return doWrite( applicationScope, entityId, version, colMutation -> colMutation.putColumn( COL_VALUE, entitySerializer.toByteBuffer(
+ new EntityWrapper( MvccEntity.Status.COMPLETE, version, Optional.<Entity>absent() ) ) ) );
}
@@ -292,12 +274,7 @@ public class MvccEntitySerializationStrategyV3Impl implements MvccEntitySerializ
Preconditions.checkNotNull( version, "version is required" );
- return doWrite( applicationScope, entityId, version, new RowOp() {
- @Override
- public void doOp( final ColumnListMutation<Boolean> colMutation ) {
- colMutation.deleteColumn( Boolean.TRUE );
- }
- } );
+ return doWrite( applicationScope, entityId, version, colMutation -> colMutation.deleteColumn( Boolean.TRUE ) );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9a89437b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
index 36faf62..68a04d0 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
@@ -754,6 +754,7 @@ public class EntityCollectionManagerIT {
//write new versions
for ( int i = 1; i < size; i++ ) {
final Entity newEntity = new Entity( entityId );
+ newEntity.setField( new IntegerField( "counter", i ) );
final Entity returnedEntity = manager.write( newEntity ).toBlocking().last();
@@ -781,7 +782,7 @@ public class EntityCollectionManagerIT {
//now get all the log versions, and delete them all we do it in 2+ batches to ensure we clean up as expected
- manager.getVersions( entityId ).buffer( 100 ).flatMap( bufferList -> manager.delete( bufferList ) )
+ manager.getVersions( entityId ).toList().flatMap( bufferList -> manager.delete( bufferList ) )
.toBlocking().last();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9a89437b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImplTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImplTest.java
index 11575c7..f6720f9 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImplTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImplTest.java
@@ -190,7 +190,7 @@ public abstract class MvccEntitySerializationStrategyImplTest {
assertEquals( version, entity.getVersion() );
- //now delete it
+ //now delete it, should remove it from cass
serializationStrategy.delete( context, id, version ).execute();
//now get it, should be gone