You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sf...@apache.org on 2015/03/20 00:26:18 UTC
[01/50] [abbrv] incubator-usergrid git commit: Updated entity
migration to run in parallel
Repository: incubator-usergrid
Updated Branches:
refs/heads/USERGRID-480 679812e6d -> 735eedb2c
Updated entity migration to run in parallel
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/95897754
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/95897754
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/95897754
Branch: refs/heads/USERGRID-480
Commit: 95897754a083debeae5fd1733836f7905baec76f
Parents: 787bc09
Author: Todd Nine <tn...@apigee.com>
Authored: Thu Feb 26 16:01:59 2015 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Thu Feb 26 16:01:59 2015 -0700
----------------------------------------------------------------------
.../impl/MvccEntityDataMigrationImpl.java | 150 +++++++++++--------
.../impl/EntityVersionCreatedTaskTest.java | 2 -
.../core/migration/data/DataMigration.java | 3 +-
.../core/scope/ApplicationEntityGroup.java | 4 +-
4 files changed, 95 insertions(+), 64 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/95897754/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntityDataMigrationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntityDataMigrationImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntityDataMigrationImpl.java
index 4c46769..4dfc64b 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntityDataMigrationImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntityDataMigrationImpl.java
@@ -19,10 +19,12 @@
*/
package org.apache.usergrid.persistence.collection.serialization.impl;
-import com.google.inject.Inject;
-import com.netflix.astyanax.Keyspace;
-import com.netflix.astyanax.MutationBatch;
-import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.UUID;
+import java.util.concurrent.atomic.AtomicLong;
+
import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.collection.EntitySet;
import org.apache.usergrid.persistence.collection.MvccEntity;
@@ -33,20 +35,20 @@ import org.apache.usergrid.persistence.core.migration.data.DataMigration;
import org.apache.usergrid.persistence.core.migration.data.DataMigrationException;
import org.apache.usergrid.persistence.core.migration.schema.MigrationStrategy;
import org.apache.usergrid.persistence.core.scope.ApplicationEntityGroup;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.scope.EntityIdScope;
-import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
+
+import com.google.inject.Inject;
+import com.netflix.astyanax.Keyspace;
+import com.netflix.astyanax.MutationBatch;
+import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
+
import rx.Observable;
import rx.functions.Action1;
import rx.functions.Func1;
+import rx.functions.Func2;
import rx.schedulers.Schedulers;
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.List;
-import java.util.UUID;
-import java.util.concurrent.atomic.AtomicLong;
/**
* Data migration strategy for entities
@@ -56,74 +58,102 @@ public class MvccEntityDataMigrationImpl implements CollectionDataMigration {
private final Keyspace keyspace;
private final MvccEntityMigrationStrategy entityMigrationStrategy;
+
@Inject
- public MvccEntityDataMigrationImpl(Keyspace keyspace, MvccEntityMigrationStrategy serializationStrategy){
+ public MvccEntityDataMigrationImpl( Keyspace keyspace, MvccEntityMigrationStrategy serializationStrategy ) {
this.keyspace = keyspace;
this.entityMigrationStrategy = serializationStrategy;
}
+
@Override
- public Observable migrate(final Observable<ApplicationEntityGroup> applicationEntityGroupObservable, final DataMigration.ProgressObserver observer) {
+ public Observable migrate( final Observable<ApplicationEntityGroup> applicationEntityGroupObservable,
+ final DataMigration.ProgressObserver observer ) {
final AtomicLong atomicLong = new AtomicLong();
- final MutationBatch totalBatch = keyspace.prepareMutationBatch();
+ //capture the time the test starts
final UUID now = UUIDGenerator.newTimeUUID();
- return applicationEntityGroupObservable.flatMap(new Func1<ApplicationEntityGroup, Observable<Id>>() {
+ return applicationEntityGroupObservable.flatMap( new Func1<ApplicationEntityGroup, Observable<Long>>() {
@Override
- public Observable call(final ApplicationEntityGroup applicationEntityGroup) {
+ public Observable<Long> call( final ApplicationEntityGroup applicationEntityGroup ) {
final List<EntityIdScope<CollectionScope>> entityIds = applicationEntityGroup.entityIds;
//go through each entity in the system, and load it's entire
// history
- return Observable.from(entityIds)
- .subscribeOn(Schedulers.io())
- .map(new Func1<EntityIdScope<CollectionScope>, Id>() {
- @Override
- public Id call(EntityIdScope<CollectionScope> idScope) {
-
- MigrationStrategy.MigrationRelationship<MvccEntitySerializationStrategy> migration = entityMigrationStrategy.getMigration();
-
-
- CollectionScope currentScope = idScope.getCollectionScope();
- //for each element in the history in the previous version,
- // copy it to the CF in v2
- EntitySet allVersions = migration.from()
- .load( currentScope, Collections.singleton( idScope.getId() ), now );
-
- final MvccEntity version = allVersions.getEntity( idScope.getId() );
-
- final MutationBatch versionBatch =
- migration.to().write(currentScope, version);
-
- totalBatch.mergeShallow(versionBatch);
-
- throw new UnsupportedOperationException( "TODO, make this more functional in flushing" );
-
-// if (atomicLong.incrementAndGet() % 50 == 0) {
-// executeBatch(totalBatch, observer, atomicLong);
-// }
-
-// executeBatch(totalBatch, observer, atomicLong);
-
-// return idScope.getId();
- }
-
-
- } ).buffer(100).doOnNext( new Action1<List<Id>>() {
- @Override
- public void call( List<Id> ids ) {
- executeBatch( totalBatch, observer, atomicLong );
- }
- });
+ return Observable.from( entityIds ).subscribeOn( Schedulers.io() )
+ .parallel( new Func1<Observable<EntityIdScope<CollectionScope>>, Observable<Long>>() {
+
+
+ //process the ids in parallel
+ @Override
+ public Observable<Long> call(
+ final Observable<EntityIdScope<CollectionScope>> entityIdScopeObservable
+ ) {
+
+ final MutationBatch totalBatch = keyspace.prepareMutationBatch();
+
+ return entityIdScopeObservable.doOnNext(
+ new Action1<EntityIdScope<CollectionScope>>() {
+
+ //load the entity and add it to the toal mutation
+ @Override
+ public void call( final EntityIdScope<CollectionScope> idScope ) {
+
+ //load the entity
+ MigrationStrategy
+ .MigrationRelationship<MvccEntitySerializationStrategy>
+ migration = entityMigrationStrategy.getMigration();
+
+
+ CollectionScope currentScope = idScope.getCollectionScope();
+ //for each element in the history in the previous
+ // version,
+ // copy it to the CF in v2
+ EntitySet allVersions = migration.from().load( currentScope,
+ Collections.singleton( idScope.getId() ), now );
+
+ final MvccEntity version =
+ allVersions.getEntity( idScope.getId() );
+
+ final MutationBatch versionBatch =
+ migration.to().write( currentScope, version );
+
+ totalBatch.mergeShallow( versionBatch );
+ }
+ } )
+ //every 100 flush the mutation
+ .buffer( 100 ).doOnNext(
+ new Action1<List<EntityIdScope<CollectionScope>>>() {
+ @Override
+ public void call(
+ final List<EntityIdScope<CollectionScope>> ids ) {
+ atomicLong.addAndGet( 100 );
+ executeBatch( totalBatch, observer, atomicLong );
+ }
+ } )
+ //count the results
+ .reduce( 0l,
+ new Func2<Long, List<EntityIdScope<CollectionScope>>, Long>() {
+ @Override
+ public Long call( final Long aLong,
+ final
+ List<EntityIdScope<CollectionScope>>
+ ids ) {
+ return aLong + ids.size();
+ }
+ } );
+ }
+ } );
}
- });
-
+ } );
}
- protected void executeBatch( final MutationBatch batch, final DataMigration.ProgressObserver po, final AtomicLong count ) {
+
+ protected void executeBatch( final MutationBatch batch, final DataMigration.ProgressObserver po,
+ final AtomicLong count ) {
try {
batch.execute();
@@ -135,11 +165,13 @@ public class MvccEntityDataMigrationImpl implements CollectionDataMigration {
}
}
+
@Override
public int getVersion() {
return entityMigrationStrategy.getVersion();
}
+
@Override
public MigrationType getType() {
return MigrationType.Entities;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/95897754/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTaskTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTaskTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTaskTest.java
index 90055a4..ade58f6 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTaskTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTaskTest.java
@@ -59,8 +59,6 @@ public class EntityVersionCreatedTaskTest {
// create a latch for the event listener, and add it to the list of events
- final int sizeToReturn = 0;
-
final Set<EntityVersionCreated> listeners = mock( Set.class );
when ( listeners.size()).thenReturn( 0 );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/95897754/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/DataMigration.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/DataMigration.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/DataMigration.java
index 5b04466..e1e3092 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/DataMigration.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/DataMigration.java
@@ -46,9 +46,10 @@ public interface DataMigration <T> {
/**
* Migrate the data to the specified version
* @param observer
+ * @Return an observable containing the count of the number of elements migrated
* @throws Throwable
*/
- public Observable migrate(final Observable<T> applicationEntityGroup,final ProgressObserver observer) throws Throwable;
+ public Observable<Long> migrate(final Observable<T> applicationEntityGroup,final ProgressObserver observer) throws Throwable;
/**
* Get the version of this migration. It must be unique.
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/95897754/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/scope/ApplicationEntityGroup.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/scope/ApplicationEntityGroup.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/scope/ApplicationEntityGroup.java
index 98a21dd..d0d81c7 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/scope/ApplicationEntityGroup.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/scope/ApplicationEntityGroup.java
@@ -28,10 +28,10 @@ import java.util.List;
* Get the entity data. Immutable bean for fast access
*/
public final class ApplicationEntityGroup<T extends ApplicationScope> {
- public final ApplicationScope applicationScope;
+ public final T applicationScope;
public final List<EntityIdScope<T>> entityIds;
- public ApplicationEntityGroup(final ApplicationScope applicationScope, final List<EntityIdScope<T>> entityIds) {
+ public ApplicationEntityGroup(final T applicationScope, final List<EntityIdScope<T>> entityIds) {
this.applicationScope = applicationScope;
this.entityIds = entityIds;
}
[11/50] [abbrv] incubator-usergrid git commit: Pushed migration tests
down to core tiers. WIP
Posted by sf...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/802dcdec/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerIT.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerIT.java
index 69e0422..ef5d69d 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerIT.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerIT.java
@@ -29,6 +29,7 @@ import org.junit.Test;
import org.apache.usergrid.persistence.core.guice.MigrationManagerRule;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
+import org.apache.usergrid.persistence.core.util.IdGenerator;
import org.apache.usergrid.persistence.graph.impl.SimpleSearchEdgeType;
import org.apache.usergrid.persistence.graph.impl.SimpleSearchIdType;
import org.apache.usergrid.persistence.model.entity.Id;
@@ -40,7 +41,7 @@ import rx.Observable;
import static org.apache.usergrid.persistence.graph.test.util.EdgeTestUtils.createEdge;
import static org.apache.usergrid.persistence.graph.test.util.EdgeTestUtils.createGetByEdge;
-import static org.apache.usergrid.persistence.graph.test.util.EdgeTestUtils.createId;
+import static org.apache.usergrid.persistence.core.util.IdGenerator.createId;
import static org.apache.usergrid.persistence.graph.test.util.EdgeTestUtils.createSearchByEdge;
import static org.apache.usergrid.persistence.graph.test.util.EdgeTestUtils.createSearchByEdgeAndId;
import static org.junit.Assert.assertEquals;
@@ -73,7 +74,7 @@ public abstract class GraphManagerIT {
@Before
public void mockApp() {
- this.scope = new ApplicationScopeImpl(createId("application") );
+ this.scope = new ApplicationScopeImpl( IdGenerator.createId( "application" ) );
}
@@ -361,18 +362,18 @@ public abstract class GraphManagerIT {
public void testWriteReadEdgeTypePagingSource() throws TimeoutException, InterruptedException {
GraphManager gm = getHelper( emf.createEdgeManager( scope ) );
- final Id sourceId = createId( "source" );
+ final Id sourceId = IdGenerator.createId( "source" );
- Edge edge1 = createEdge( sourceId, "test", createId( "target" ) );
+ Edge edge1 = createEdge( sourceId, "test", IdGenerator.createId( "target" ) );
gm.writeEdge( edge1 ).toBlocking().last();
- Edge edge2 = createEdge( sourceId, "test", createId( "target" ) );
+ Edge edge2 = createEdge( sourceId, "test", IdGenerator.createId( "target" ) );
gm.writeEdge( edge2 ).toBlocking().last();
- Edge edge3 = createEdge( sourceId, "test", createId( "target" ) );
+ Edge edge3 = createEdge( sourceId, "test", IdGenerator.createId( "target" ) );
gm.writeEdge( edge3 ).toBlocking().last();
@@ -418,17 +419,17 @@ public abstract class GraphManagerIT {
GraphManager gm = getHelper( emf.createEdgeManager( scope ) );
- final Id targetId = createId( "target" );
+ final Id targetId = IdGenerator.createId( "target" );
- Edge edge1 = createEdge( createId( "source" ), "test", targetId );
+ Edge edge1 = createEdge( IdGenerator.createId( "source" ), "test", targetId );
gm.writeEdge( edge1 ).toBlocking().last();
- Edge edge2 = createEdge( createId( "source" ), "test", targetId );
+ Edge edge2 = createEdge( IdGenerator.createId( "source" ), "test", targetId );
gm.writeEdge( edge2 ).toBlocking().last();
- Edge edge3 = createEdge( createId( "source" ), "test", targetId );
+ Edge edge3 = createEdge( IdGenerator.createId( "source" ), "test", targetId );
gm.writeEdge( edge3 ).toBlocking().last();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/802dcdec/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerLoadTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerLoadTest.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerLoadTest.java
index d3bf24e..976a6cc 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerLoadTest.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerLoadTest.java
@@ -44,6 +44,7 @@ import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
import org.apache.usergrid.persistence.core.test.ITRunner;
import org.apache.usergrid.persistence.core.test.UseModules;
+import org.apache.usergrid.persistence.core.util.IdGenerator;
import org.apache.usergrid.persistence.graph.guice.TestGraphModule;
import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdgeType;
import org.apache.usergrid.persistence.model.entity.Id;
@@ -54,7 +55,7 @@ import rx.Observable;
import rx.Subscriber;
import static org.apache.usergrid.persistence.graph.test.util.EdgeTestUtils.createEdge;
-import static org.apache.usergrid.persistence.graph.test.util.EdgeTestUtils.createId;
+import static org.apache.usergrid.persistence.core.util.IdGenerator.createId;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.fail;
@@ -85,7 +86,7 @@ public class GraphManagerLoadTest {
//get the system property of the UUID to use. If one is not set, use the defualt
String uuidString = System.getProperty( "org.id", "80a42760-b699-11e3-a5e2-0800200c9a66" );
- scope = new ApplicationScopeImpl( createId( UUID.fromString( uuidString ), "test" ) );
+ scope = new ApplicationScopeImpl( IdGenerator.createId( UUID.fromString( uuidString ), "test" ) );
numWorkers = Integer.parseInt( System.getProperty( "numWorkers", "100" ) );
writeLimit = Integer.parseInt( System.getProperty( "writeLimit", "10000" ) );
@@ -98,12 +99,12 @@ public class GraphManagerLoadTest {
public void writeThousandsSingleSource() throws InterruptedException, ExecutionException {
EdgeGenerator generator = new EdgeGenerator() {
- private Id sourceId = createId( "source" );
+ private Id sourceId = IdGenerator.createId( "source" );
@Override
public Edge newEdge() {
- Edge edge = createEdge( sourceId, "test", createId( "target" ) );
+ Edge edge = createEdge( sourceId, "test", IdGenerator.createId( "target" ) );
return edge;
@@ -125,12 +126,12 @@ public class GraphManagerLoadTest {
public void writeThousandsSingleTarget() throws InterruptedException, ExecutionException {
EdgeGenerator generator = new EdgeGenerator() {
- private Id targetId = createId( "target" );
+ private Id targetId = IdGenerator.createId( "target" );
@Override
public Edge newEdge() {
- Edge edge = createEdge( createId( "source" ), "test", targetId );
+ Edge edge = createEdge( IdGenerator.createId( "source" ), "test", targetId );
return edge;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/802dcdec/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardConsistencyIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardConsistencyIT.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardConsistencyIT.java
index 095f855..6d30d22 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardConsistencyIT.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardConsistencyIT.java
@@ -41,7 +41,6 @@ import javax.annotation.Nullable;
import org.junit.After;
import org.junit.Before;
-import org.junit.Ignore;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -50,6 +49,7 @@ import org.apache.usergrid.persistence.core.migration.schema.MigrationException;
import org.apache.usergrid.persistence.core.migration.schema.MigrationManager;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
+import org.apache.usergrid.persistence.core.util.IdGenerator;
import org.apache.usergrid.persistence.graph.guice.TestGraphModule;
import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdgeType;
import org.apache.usergrid.persistence.graph.serialization.impl.shard.DirectedEdgeMeta;
@@ -73,7 +73,7 @@ import rx.Observable;
import rx.functions.Action1;
import static org.apache.usergrid.persistence.graph.test.util.EdgeTestUtils.createEdge;
-import static org.apache.usergrid.persistence.graph.test.util.EdgeTestUtils.createId;
+import static org.apache.usergrid.persistence.core.util.IdGenerator.createId;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.fail;
@@ -129,7 +129,7 @@ public class GraphManagerShardConsistencyIT {
//get the system property of the UUID to use. If one is not set, use the defualt
String uuidString = System.getProperty( "org.id", "80a42760-b699-11e3-a5e2-0800200c9a66" );
- scope = new ApplicationScopeImpl( createId( UUID.fromString( uuidString ), "test" ) );
+ scope = new ApplicationScopeImpl( IdGenerator.createId( UUID.fromString( uuidString ), "test" ) );
reporter.start( 10, TimeUnit.SECONDS );
@@ -147,7 +147,7 @@ public class GraphManagerShardConsistencyIT {
public void writeThousandsSingleSource()
throws InterruptedException, ExecutionException, MigrationException, UnsupportedEncodingException {
- final Id sourceId = createId( "source" );
+ final Id sourceId = IdGenerator.createId( "source" );
final String edgeType = "test";
final EdgeGenerator generator = new EdgeGenerator() {
@@ -155,7 +155,7 @@ public class GraphManagerShardConsistencyIT {
@Override
public Edge newEdge() {
- Edge edge = createEdge( sourceId, edgeType, createId( "target" ) );
+ Edge edge = createEdge( sourceId, edgeType, IdGenerator.createId( "target" ) );
return edge;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/802dcdec/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardingIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardingIT.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardingIT.java
index a9035d3..2951efe 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardingIT.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardingIT.java
@@ -34,6 +34,7 @@ import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
import org.apache.usergrid.persistence.core.test.ITRunner;
import org.apache.usergrid.persistence.core.test.UseModules;
+import org.apache.usergrid.persistence.core.util.IdGenerator;
import org.apache.usergrid.persistence.graph.guice.TestGraphModule;
import org.apache.usergrid.persistence.graph.serialization.impl.shard.DirectedEdgeMeta;
import org.apache.usergrid.persistence.graph.serialization.impl.shard.NodeShardApproximation;
@@ -43,7 +44,7 @@ import org.apache.usergrid.persistence.model.entity.Id;
import com.google.inject.Inject;
import static org.apache.usergrid.persistence.graph.test.util.EdgeTestUtils.createEdge;
-import static org.apache.usergrid.persistence.graph.test.util.EdgeTestUtils.createId;
+import static org.apache.usergrid.persistence.core.util.IdGenerator.createId;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@@ -76,7 +77,7 @@ public class GraphManagerShardingIT {
@Before
public void mockApp() {
- this.scope = new ApplicationScopeImpl(createId("application") );
+ this.scope = new ApplicationScopeImpl( IdGenerator.createId( "application" ) );
}
@@ -85,7 +86,7 @@ public class GraphManagerShardingIT {
GraphManager gm = emf.createEdgeManager( scope ) ;
- final Id sourceId = createId( "source" );
+ final Id sourceId = IdGenerator.createId( "source" );
final String edgeType = "test";
@@ -107,7 +108,7 @@ public class GraphManagerShardingIT {
Id targetId = null;
for(long i = 0; i < writeCount; i ++){
- targetId = createId("target") ;
+ targetId = IdGenerator.createId( "target" ) ;
final Edge edge = createEdge( sourceId, edgeType, targetId);
@@ -143,7 +144,7 @@ public class GraphManagerShardingIT {
GraphManager gm = emf.createEdgeManager( scope ) ;
- final Id targetId = createId( "target" );
+ final Id targetId = IdGenerator.createId( "target" );
final String edgeType = "test";
@@ -161,7 +162,7 @@ public class GraphManagerShardingIT {
Id sourceId = null;
for(long i = 0; i < writeCount; i ++){
- sourceId = createId("source") ;
+ sourceId = IdGenerator.createId( "source" ) ;
final Edge edge = createEdge( sourceId, edgeType, targetId);
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/802dcdec/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerStressTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerStressTest.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerStressTest.java
index cc131d1..9380352 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerStressTest.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerStressTest.java
@@ -37,6 +37,7 @@ import org.apache.usergrid.persistence.core.guice.MigrationManagerRule;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.test.ITRunner;
import org.apache.usergrid.persistence.core.test.UseModules;
+import org.apache.usergrid.persistence.core.util.IdGenerator;
import org.apache.usergrid.persistence.graph.guice.TestGraphModule;
import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdgeType;
import org.apache.usergrid.persistence.model.entity.Id;
@@ -48,7 +49,7 @@ import rx.Observable;
import rx.Subscriber;
import static org.apache.usergrid.persistence.graph.test.util.EdgeTestUtils.createEdge;
-import static org.apache.usergrid.persistence.graph.test.util.EdgeTestUtils.createId;
+import static org.apache.usergrid.persistence.core.util.IdGenerator.createId;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.fail;
@@ -179,12 +180,12 @@ public class GraphManagerStressTest {
public void writeThousandsSingleSource() throws InterruptedException {
EdgeGenerator generator = new EdgeGenerator() {
- private Id sourceId = createId( "source" );
+ private Id sourceId = IdGenerator.createId( "source" );
@Override
public Edge newEdge() {
- Edge edge = createEdge( sourceId, "test", createId( "target" ) );
+ Edge edge = createEdge( sourceId, "test", IdGenerator.createId( "target" ) );
return edge;
@@ -206,12 +207,12 @@ public class GraphManagerStressTest {
public void writeThousandsSingleTarget() throws InterruptedException {
EdgeGenerator generator = new EdgeGenerator() {
- private Id targetId = createId( "target" );
+ private Id targetId = IdGenerator.createId( "target" );
@Override
public Edge newEdge() {
- Edge edge = createEdge( createId( "source" ), "test", targetId );
+ Edge edge = createEdge( IdGenerator.createId( "source" ), "test", targetId );
return edge;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/802dcdec/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/guice/TestGraphModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/guice/TestGraphModule.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/guice/TestGraphModule.java
index 7dd2254..3201a92 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/guice/TestGraphModule.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/guice/TestGraphModule.java
@@ -24,6 +24,7 @@ import org.apache.usergrid.persistence.core.guice.TestModule;
import org.apache.usergrid.persistence.core.migration.data.newimpls.MigrationDataProvider;
import org.apache.usergrid.persistence.core.migration.data.newimpls.TestMigrationDataProvider;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.graph.serialization.impl.migration.GraphNode;
import com.google.inject.TypeLiteral;
@@ -45,8 +46,8 @@ public class TestGraphModule extends TestModule {
public void configureMigrationProvider() {
//configure our migration data provider
- TestMigrationDataProvider<ApplicationScope> migrationDataProvider = new TestMigrationDataProvider<>();
- bind( new TypeLiteral<MigrationDataProvider<ApplicationScope>>() {} )
+ TestMigrationDataProvider<GraphNode> migrationDataProvider = new TestMigrationDataProvider<>();
+ bind( new TypeLiteral<MigrationDataProvider<GraphNode>>() {} )
.toInstance( migrationDataProvider );
}
} );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/802dcdec/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/impl/EdgeDeleteListenerTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/impl/EdgeDeleteListenerTest.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/impl/EdgeDeleteListenerTest.java
index b8c5b3e..84783e5 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/impl/EdgeDeleteListenerTest.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/impl/EdgeDeleteListenerTest.java
@@ -36,6 +36,7 @@ import org.apache.usergrid.persistence.core.guice.ProxyImpl;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.test.ITRunner;
import org.apache.usergrid.persistence.core.test.UseModules;
+import org.apache.usergrid.persistence.core.util.IdGenerator;
import org.apache.usergrid.persistence.graph.GraphFig;
import org.apache.usergrid.persistence.graph.GraphManagerFactory;
import org.apache.usergrid.persistence.graph.MarkedEdge;
@@ -50,7 +51,7 @@ import com.google.inject.Inject;
import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
import static org.apache.usergrid.persistence.graph.test.util.EdgeTestUtils.createGetByEdge;
-import static org.apache.usergrid.persistence.graph.test.util.EdgeTestUtils.createId;
+import static org.apache.usergrid.persistence.core.util.IdGenerator.createId;
import static org.apache.usergrid.persistence.graph.test.util.EdgeTestUtils.createMarkedEdge;
import static org.apache.usergrid.persistence.graph.test.util.EdgeTestUtils.createSearchByEdge;
import static org.apache.usergrid.persistence.graph.test.util.EdgeTestUtils.createSearchByEdgeAndId;
@@ -113,9 +114,9 @@ public class EdgeDeleteListenerTest {
//write several versions to the commit log
- final Id sourceId = createId( "source" );
+ final Id sourceId = IdGenerator.createId( "source" );
final String edgeType = "test";
- final Id targetId = createId( "target" );
+ final Id targetId = IdGenerator.createId( "target" );
final long edgeTimestamp = 1000l;
@@ -206,9 +207,9 @@ public class EdgeDeleteListenerTest {
//write several versions to the commit log
- final Id sourceId = createId( "source" );
+ final Id sourceId = IdGenerator.createId( "source" );
final String edgeType = "test";
- final Id targetId = createId( "target" );
+ final Id targetId = IdGenerator.createId( "target" );
final long timestamp = 1000l;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/802dcdec/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/impl/NodeDeleteListenerTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/impl/NodeDeleteListenerTest.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/impl/NodeDeleteListenerTest.java
index d91bfc3..ad433e9 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/impl/NodeDeleteListenerTest.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/impl/NodeDeleteListenerTest.java
@@ -35,6 +35,7 @@ import org.apache.usergrid.persistence.core.guice.ProxyImpl;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.test.ITRunner;
import org.apache.usergrid.persistence.core.test.UseModules;
+import org.apache.usergrid.persistence.core.util.IdGenerator;
import org.apache.usergrid.persistence.graph.Edge;
import org.apache.usergrid.persistence.graph.GraphFig;
import org.apache.usergrid.persistence.graph.GraphManager;
@@ -53,7 +54,7 @@ import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
import static org.apache.usergrid.persistence.graph.test.util.EdgeTestUtils.createEdge;
import static org.apache.usergrid.persistence.graph.test.util.EdgeTestUtils.createGetByEdge;
-import static org.apache.usergrid.persistence.graph.test.util.EdgeTestUtils.createId;
+import static org.apache.usergrid.persistence.core.util.IdGenerator.createId;
import static org.apache.usergrid.persistence.graph.test.util.EdgeTestUtils.createSearchByEdge;
import static org.apache.usergrid.persistence.graph.test.util.EdgeTestUtils.createSearchEdge;
import static org.apache.usergrid.persistence.graph.test.util.EdgeTestUtils.createSearchIdType;
@@ -329,7 +330,7 @@ public class NodeDeleteListenerTest {
//create loads of edges to easily delete. We'll keep all the types of "test"
final int edgeCount = graphFig.getScanPageSize() * 2;
- Id toDelete = createId( "toDelete" );
+ Id toDelete = IdGenerator.createId( "toDelete" );
final String edgeType = "test";
int countSaved = 0;
@@ -342,11 +343,11 @@ public class NodeDeleteListenerTest {
//mix up source vs target, good for testing as well as create a lot of sub types to ensure they're removed
if ( i % 2 == 0 ) {
- edge = createEdge( toDelete, edgeType, createId( "target" + Math.random() ) );
+ edge = createEdge( toDelete, edgeType, IdGenerator.createId( "target" + Math.random() ) );
sourceCount++;
}
else {
- edge = createEdge( createId( "source" + Math.random() ), edgeType, toDelete );
+ edge = createEdge( IdGenerator.createId( "source" + Math.random() ), edgeType, toDelete );
targetCount++;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/802dcdec/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeDeleteRepairTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeDeleteRepairTest.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeDeleteRepairTest.java
index cdb284d..238daa0 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeDeleteRepairTest.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeDeleteRepairTest.java
@@ -33,6 +33,7 @@ import org.apache.usergrid.persistence.core.guice.MigrationManagerRule;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.test.ITRunner;
import org.apache.usergrid.persistence.core.test.UseModules;
+import org.apache.usergrid.persistence.core.util.IdGenerator;
import org.apache.usergrid.persistence.graph.MarkedEdge;
import org.apache.usergrid.persistence.graph.SearchByEdgeType;
import org.apache.usergrid.persistence.graph.guice.TestGraphModule;
@@ -45,7 +46,7 @@ import com.google.inject.Inject;
import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
import static org.apache.usergrid.persistence.graph.test.util.EdgeTestUtils.createEdge;
-import static org.apache.usergrid.persistence.graph.test.util.EdgeTestUtils.createId;
+import static org.apache.usergrid.persistence.core.util.IdGenerator.createId;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.mockito.Mockito.mock;
@@ -113,8 +114,8 @@ public class EdgeDeleteRepairTest {
public void commitLogTest() throws ConnectionException {
- final Id sourceId = createId( "source" );
- final Id targetId = createId( "target" );
+ final Id sourceId = IdGenerator.createId( "source" );
+ final Id targetId = IdGenerator.createId( "target" );
final String edgeType = "edge";
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/802dcdec/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeMetaRepairTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeMetaRepairTest.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeMetaRepairTest.java
index 1b57c2d..f729f33 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeMetaRepairTest.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeMetaRepairTest.java
@@ -34,6 +34,7 @@ import org.apache.usergrid.persistence.core.guice.ProxyImpl;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.test.ITRunner;
import org.apache.usergrid.persistence.core.test.UseModules;
+import org.apache.usergrid.persistence.core.util.IdGenerator;
import org.apache.usergrid.persistence.graph.GraphFig;
import org.apache.usergrid.persistence.graph.MarkedEdge;
import org.apache.usergrid.persistence.graph.guice.TestGraphModule;
@@ -48,7 +49,7 @@ import com.google.inject.Inject;
import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
import static org.apache.usergrid.persistence.graph.test.util.EdgeTestUtils.createEdge;
-import static org.apache.usergrid.persistence.graph.test.util.EdgeTestUtils.createId;
+import static org.apache.usergrid.persistence.core.util.IdGenerator.createId;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.mockito.Mockito.mock;
@@ -102,7 +103,7 @@ public class EdgeMetaRepairTest {
public void cleanTargetNoEdgesNoMeta() {
//do no writes, then execute a cleanup with no meta data
- final Id targetId = createId( "target" );
+ final Id targetId = IdGenerator.createId( "target" );
final String test = "test";
final long version = System.currentTimeMillis();
@@ -152,22 +153,22 @@ public class EdgeMetaRepairTest {
@Test
public void cleanTargetMultipleEdge() throws ConnectionException {
- Id targetId = createId( "target" );
+ Id targetId = IdGenerator.createId( "target" );
- MarkedEdge edge1 = createEdge( createId( "source1" ), "test", targetId );
+ MarkedEdge edge1 = createEdge( IdGenerator.createId( "source1" ), "test", targetId );
storageEdgeSerialization.writeEdge( scope, edge1, UUIDGenerator.newTimeUUID() ).execute();
edgeMetadataSerialization.writeEdge( scope, edge1 ).execute();
- MarkedEdge edge2 = createEdge( createId( "source2" ), "test", targetId );
+ MarkedEdge edge2 = createEdge( IdGenerator.createId( "source2" ), "test", targetId );
storageEdgeSerialization.writeEdge( scope, edge2, UUIDGenerator.newTimeUUID() ).execute();
edgeMetadataSerialization.writeEdge( scope, edge2 ).execute();
- MarkedEdge edge3 = createEdge( createId( "source3" ), "test", targetId );
+ MarkedEdge edge3 = createEdge( IdGenerator.createId( "source3" ), "test", targetId );
storageEdgeSerialization.writeEdge( scope, edge3, UUIDGenerator.newTimeUUID() ).execute();
@@ -223,7 +224,7 @@ public class EdgeMetaRepairTest {
@Test
public void cleanTargetMultipleEdgeBuffer() throws ConnectionException {
- final Id targetId = createId( "target" );
+ final Id targetId = IdGenerator.createId( "target" );
final String edgeType = "test";
final int size = graphFig.getRepairConcurrentSize() * 2;
@@ -232,7 +233,7 @@ public class EdgeMetaRepairTest {
for ( int i = 0; i < size; i++ ) {
- MarkedEdge edge = createEdge( createId( "source" + i ), edgeType, targetId );
+ MarkedEdge edge = createEdge( IdGenerator.createId( "source" + i ), edgeType, targetId );
storageEdgeSerialization.writeEdge( scope, edge, UUIDGenerator.newTimeUUID() ).execute();
@@ -315,22 +316,22 @@ public class EdgeMetaRepairTest {
@Test
public void cleanSourceMultipleEdge() throws ConnectionException {
- Id sourceId = createId( "source" );
+ Id sourceId = IdGenerator.createId( "source" );
- MarkedEdge edge1 = createEdge( sourceId, "test", createId( "target1" ) );
+ MarkedEdge edge1 = createEdge( sourceId, "test", IdGenerator.createId( "target1" ) );
storageEdgeSerialization.writeEdge( scope, edge1, UUIDGenerator.newTimeUUID() ).execute();
edgeMetadataSerialization.writeEdge( scope, edge1 ).execute();
- MarkedEdge edge2 = createEdge( sourceId, "test", createId( "target2" ) );
+ MarkedEdge edge2 = createEdge( sourceId, "test", IdGenerator.createId( "target2" ) );
storageEdgeSerialization.writeEdge( scope, edge2, UUIDGenerator.newTimeUUID() ).execute();
edgeMetadataSerialization.writeEdge( scope, edge2).execute();
- MarkedEdge edge3 = createEdge( sourceId, "test", createId( "target3" ) );
+ MarkedEdge edge3 = createEdge( sourceId, "test", IdGenerator.createId( "target3" ) );
storageEdgeSerialization.writeEdge( scope, edge3, UUIDGenerator.newTimeUUID() ).execute();
@@ -386,7 +387,7 @@ public class EdgeMetaRepairTest {
@Test
public void cleanSourceMultipleEdgeBuffer() throws ConnectionException {
- Id sourceId = createId( "source" );
+ Id sourceId = IdGenerator.createId( "source" );
final String edgeType = "test";
@@ -396,7 +397,7 @@ public class EdgeMetaRepairTest {
for ( int i = 0; i < size; i++ ) {
- MarkedEdge edge = createEdge( sourceId, edgeType, createId( "target" + i ) );
+ MarkedEdge edge = createEdge( sourceId, edgeType, IdGenerator.createId( "target" + i ) );
storageEdgeSerialization.writeEdge( scope, edge, UUIDGenerator.newTimeUUID() ).execute();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/802dcdec/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/EdgeMetadataSerializationTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/EdgeMetadataSerializationTest.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/EdgeMetadataSerializationTest.java
index 1ed1742..f5b8e89 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/EdgeMetadataSerializationTest.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/EdgeMetadataSerializationTest.java
@@ -29,6 +29,7 @@ import org.junit.Test;
import org.apache.usergrid.persistence.core.guice.MigrationManagerRule;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.core.util.IdGenerator;
import org.apache.usergrid.persistence.graph.Edge;
import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
@@ -44,7 +45,7 @@ import com.netflix.astyanax.model.ColumnFamily;
import com.netflix.astyanax.serializers.StringSerializer;
import static org.apache.usergrid.persistence.graph.test.util.EdgeTestUtils.createEdge;
-import static org.apache.usergrid.persistence.graph.test.util.EdgeTestUtils.createId;
+import static org.apache.usergrid.persistence.core.util.IdGenerator.createId;
import static org.apache.usergrid.persistence.graph.test.util.EdgeTestUtils.createSearchEdge;
import static org.apache.usergrid.persistence.graph.test.util.EdgeTestUtils.createSearchIdType;
import static org.junit.Assert.assertEquals;
@@ -98,9 +99,9 @@ public abstract class EdgeMetadataSerializationTest {
final Id sourceId = edge1.getSourceNode();
- final Edge edge2 = createEdge( sourceId, "edge", createId( "target2" ) );
+ final Edge edge2 = createEdge( sourceId, "edge", IdGenerator.createId( "target2" ) );
- final Edge edge3 = createEdge( sourceId, "edge2", createId( "target3" ) );
+ final Edge edge3 = createEdge( sourceId, "edge2", IdGenerator.createId( "target3" ) );
//set writing the edge
serialization.writeEdge( scope, edge1 ).execute();
@@ -132,9 +133,9 @@ public abstract class EdgeMetadataSerializationTest {
final Id targetId = edge1.getTargetNode();
- final Edge edge2 = createEdge( createId( "source" ), "edge", targetId );
+ final Edge edge2 = createEdge( IdGenerator.createId( "source" ), "edge", targetId );
- final Edge edge3 = createEdge( createId( "source2" ), "edge2", targetId );
+ final Edge edge3 = createEdge( IdGenerator.createId( "source2" ), "edge2", targetId );
//set writing the edge
serialization.writeEdge( scope, edge1 ).execute();
@@ -166,12 +167,12 @@ public abstract class EdgeMetadataSerializationTest {
final Id sourceId = edge1.getSourceNode();
- final Edge edge2 = createEdge( sourceId, "edge", createId( "target" ) );
+ final Edge edge2 = createEdge( sourceId, "edge", IdGenerator.createId( "target" ) );
- final Edge edge3 = createEdge( sourceId, "edge", createId( "target2" ) );
+ final Edge edge3 = createEdge( sourceId, "edge", IdGenerator.createId( "target2" ) );
//shouldn't be returned
- final Edge edge4 = createEdge( sourceId, "edge2", createId( "target3" ) );
+ final Edge edge4 = createEdge( sourceId, "edge2", IdGenerator.createId( "target3" ) );
//set writing the edge
serialization.writeEdge( scope, edge1 ).execute();
@@ -212,12 +213,12 @@ public abstract class EdgeMetadataSerializationTest {
final Id targetId = edge1.getTargetNode();
- final Edge edge2 = createEdge( createId( "source" ), "edge", targetId );
+ final Edge edge2 = createEdge( IdGenerator.createId( "source" ), "edge", targetId );
- final Edge edge3 = createEdge( createId( "source2" ), "edge", targetId );
+ final Edge edge3 = createEdge( IdGenerator.createId( "source2" ), "edge", targetId );
//shouldn't be returned
- final Edge edge4 = createEdge( createId( "source3" ), "edge2", targetId );
+ final Edge edge4 = createEdge( IdGenerator.createId( "source3" ), "edge2", targetId );
//set writing the edge
serialization.writeEdge( scope, edge1 ).execute();
@@ -259,9 +260,9 @@ public abstract class EdgeMetadataSerializationTest {
final Id sourceId = edge1.getSourceNode();
- final Edge edge2 = createEdge( sourceId, "edge", createId( "target2" ), timestamp + 1 );
+ final Edge edge2 = createEdge( sourceId, "edge", IdGenerator.createId( "target2" ), timestamp + 1 );
- final Edge edge3 = createEdge( sourceId, "edge2", createId( "target3" ), timestamp + 2 );
+ final Edge edge3 = createEdge( sourceId, "edge2", IdGenerator.createId( "target3" ), timestamp + 2 );
//set writing the edge
serialization.writeEdge( scope, edge1 ).execute();
@@ -312,9 +313,9 @@ public abstract class EdgeMetadataSerializationTest {
final Id targetId = edge1.getTargetNode();
- final Edge edge2 = createEdge( createId( "source" ), "edge", targetId );
+ final Edge edge2 = createEdge( IdGenerator.createId( "source" ), "edge", targetId );
- final Edge edge3 = createEdge( createId( "source2" ), "edge2", targetId );
+ final Edge edge3 = createEdge( IdGenerator.createId( "source2" ), "edge2", targetId );
//set writing the edge
serialization.writeEdge( scope, edge1 ).execute();
@@ -368,9 +369,9 @@ public abstract class EdgeMetadataSerializationTest {
final Id sourceId = edge1.getSourceNode();
- final Edge edge2 = createEdge( sourceId, "edge", createId( "target" ), timestamp + 1 );
+ final Edge edge2 = createEdge( sourceId, "edge", IdGenerator.createId( "target" ), timestamp + 1 );
- final Edge edge3 = createEdge( sourceId, "edge", createId( "target2" ), timestamp + 2 );
+ final Edge edge3 = createEdge( sourceId, "edge", IdGenerator.createId( "target2" ), timestamp + 2 );
//set writing the edge
serialization.writeEdge( scope, edge1 ).execute();
@@ -425,9 +426,9 @@ public abstract class EdgeMetadataSerializationTest {
final Id targetId = edge1.getTargetNode();
- final Edge edge2 = createEdge( createId( "source" ), "edge", targetId, timestamp + 1 );
+ final Edge edge2 = createEdge( IdGenerator.createId( "source" ), "edge", targetId, timestamp + 1 );
- final Edge edge3 = createEdge( createId( "source2" ), "edge", targetId, timestamp + 2 );
+ final Edge edge3 = createEdge( IdGenerator.createId( "source2" ), "edge", targetId, timestamp + 2 );
//set writing the edge
serialization.writeEdge( scope, edge1 ).execute();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/802dcdec/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/EdgeSerializationChopTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/EdgeSerializationChopTest.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/EdgeSerializationChopTest.java
index b1bcf2d..1197a74 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/EdgeSerializationChopTest.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/EdgeSerializationChopTest.java
@@ -31,6 +31,7 @@ import org.apache.usergrid.persistence.core.guice.MigrationManagerRule;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.test.ITRunner;
import org.apache.usergrid.persistence.core.test.UseModules;
+import org.apache.usergrid.persistence.core.util.IdGenerator;
import org.apache.usergrid.persistence.graph.MarkedEdge;
import org.apache.usergrid.persistence.graph.guice.TestGraphModule;
import org.apache.usergrid.persistence.model.entity.Id;
@@ -40,7 +41,7 @@ import com.google.inject.Inject;
import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
import static org.apache.usergrid.persistence.graph.test.util.EdgeTestUtils.createEdge;
-import static org.apache.usergrid.persistence.graph.test.util.EdgeTestUtils.createId;
+import static org.apache.usergrid.persistence.core.util.IdGenerator.createId;
import static org.apache.usergrid.persistence.graph.test.util.EdgeTestUtils.createSearchByEdge;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.mock;
@@ -100,8 +101,8 @@ public class EdgeSerializationChopTest {
public void mixedEdgeTypes() throws ConnectionException {
- final Id sourceId = createId( SOURCE_NODE_ID, "source" );
- final Id targetId = createId( "target" );
+ final Id sourceId = IdGenerator.createId( SOURCE_NODE_ID, "source" );
+ final Id targetId = IdGenerator.createId( "target" );
final MarkedEdge edge = createEdge( sourceId, "edge", targetId );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/802dcdec/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/EdgeSerializationTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/EdgeSerializationTest.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/EdgeSerializationTest.java
index 34aea46..d875e83 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/EdgeSerializationTest.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/EdgeSerializationTest.java
@@ -34,6 +34,7 @@ import org.slf4j.LoggerFactory;
import org.apache.usergrid.persistence.core.guice.MigrationManagerRule;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.core.util.IdGenerator;
import org.apache.usergrid.persistence.graph.Edge;
import org.apache.usergrid.persistence.graph.GraphFig;
import org.apache.usergrid.persistence.graph.MarkedEdge;
@@ -54,7 +55,7 @@ import com.netflix.astyanax.serializers.StringSerializer;
import static org.apache.usergrid.persistence.graph.test.util.EdgeTestUtils.createEdge;
import static org.apache.usergrid.persistence.graph.test.util.EdgeTestUtils.createGetByEdge;
-import static org.apache.usergrid.persistence.graph.test.util.EdgeTestUtils.createId;
+import static org.apache.usergrid.persistence.core.util.IdGenerator.createId;
import static org.apache.usergrid.persistence.graph.test.util.EdgeTestUtils.createMarkedEdge;
import static org.apache.usergrid.persistence.graph.test.util.EdgeTestUtils.createSearchByEdge;
import static org.apache.usergrid.persistence.graph.test.util.EdgeTestUtils.createSearchByEdgeAndId;
@@ -226,7 +227,7 @@ public abstract class EdgeSerializationTest {
final MarkedEdge edgev2 = createEdge( sourceId, "edge1", targetId, timestamp + 1 );
//we shouldn't get this one back
- final MarkedEdge diffTarget = createEdge( sourceId, "edge1", createId( "newTarget" ) );
+ final MarkedEdge diffTarget = createEdge( sourceId, "edge1", IdGenerator.createId( "newTarget" ) );
assertTrue( "Edge version 1 has lower time uuid",
Long.compare( edgev1.getTimestamp(), edgev2.getTimestamp() ) < 0 );
@@ -275,7 +276,7 @@ public abstract class EdgeSerializationTest {
final Id targetId1 = edge1.getTargetNode();
- final MarkedEdge edge2 = createEdge( sourceId, "edge", createId( "target2" ) );
+ final MarkedEdge edge2 = createEdge( sourceId, "edge", IdGenerator.createId( "target2" ) );
final Id targetId2 = edge2.getTargetNode();
@@ -329,7 +330,7 @@ public abstract class EdgeSerializationTest {
final Id targetId1 = edge1.getTargetNode();
- final MarkedEdge edge2 = createEdge( sourceId, "edge", createId( "target" ), timestamp + 1 );
+ final MarkedEdge edge2 = createEdge( sourceId, "edge", IdGenerator.createId( "target" ), timestamp + 1 );
final Id targetId2 = edge2.getTargetNode();
@@ -390,7 +391,7 @@ public abstract class EdgeSerializationTest {
final Id targetId1 = edge1.getTargetNode();
- final MarkedEdge edge2 = createEdge( sourceId, "edge", createId( "target" ), timestamp );
+ final MarkedEdge edge2 = createEdge( sourceId, "edge", IdGenerator.createId( "target" ), timestamp );
final Id targetId2 = edge2.getTargetNode();
@@ -498,7 +499,7 @@ public abstract class EdgeSerializationTest {
final Id targetId1 = edge1.getTargetNode();
- final MarkedEdge edge2 = createEdge( sourceId, "edge", createId( "target" ), timestamp + 1 );
+ final MarkedEdge edge2 = createEdge( sourceId, "edge", IdGenerator.createId( "target" ), timestamp + 1 );
final Id targetId2 = edge2.getTargetNode();
@@ -654,7 +655,7 @@ public abstract class EdgeSerializationTest {
int size = graphFig.getScanPageSize() * 2;
- final Id sourceId = createId( "source" );
+ final Id sourceId = IdGenerator.createId( "source" );
final String type = "edge";
Set<Edge> edges = new HashSet<Edge>( size );
@@ -663,7 +664,7 @@ public abstract class EdgeSerializationTest {
long timestamp = 0;
for ( int i = 0; i < size; i++ ) {
- final MarkedEdge edge = createEdge( sourceId, type, createId( "target" ), timestamp );
+ final MarkedEdge edge = createEdge( sourceId, type, IdGenerator.createId( "target" ), timestamp );
serialization.writeEdge( scope, edge, UUIDGenerator.newTimeUUID() ).execute();
edges.add( edge );
@@ -693,9 +694,9 @@ public abstract class EdgeSerializationTest {
public void testIteratorPaging() throws ConnectionException {
- final Id sourceId = createId( "source" );
+ final Id sourceId = IdGenerator.createId( "source" );
final String edgeType = "edge";
- final Id targetId = createId( "target" );
+ final Id targetId = IdGenerator.createId( "target" );
int writeCount = graphFig.getScanPageSize() * 3;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/802dcdec/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/NodeSerializationTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/NodeSerializationTest.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/NodeSerializationTest.java
index d74eb4e..0480b99 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/NodeSerializationTest.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/NodeSerializationTest.java
@@ -31,6 +31,7 @@ import org.apache.usergrid.persistence.core.guice.MigrationManagerRule;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.test.ITRunner;
import org.apache.usergrid.persistence.core.test.UseModules;
+import org.apache.usergrid.persistence.core.util.IdGenerator;
import org.apache.usergrid.persistence.graph.guice.TestGraphModule;
import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
@@ -40,7 +41,7 @@ import com.google.inject.Inject;
import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
import static org.apache.usergrid.persistence.graph.test.util.EdgeTestUtils.createEdge;
-import static org.apache.usergrid.persistence.graph.test.util.EdgeTestUtils.createId;
+import static org.apache.usergrid.persistence.core.util.IdGenerator.createId;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.mockito.Mockito.mock;
@@ -85,7 +86,7 @@ public class NodeSerializationTest {
@Test
public void writeReadDelete() throws ConnectionException {
- final Id nodeId = createId( "test" );
+ final Id nodeId = IdGenerator.createId( "test" );
final long version = System.currentTimeMillis();
serialization.mark( scope, nodeId, version ).execute();
@@ -111,7 +112,7 @@ public class NodeSerializationTest {
@Test
public void noDeleteVersion() {
- final Id nodeId = createId( "test" );
+ final Id nodeId = IdGenerator.createId( "test" );
Optional<Long> returned = serialization.getMaxVersion( scope, nodeId );
@@ -128,7 +129,7 @@ public class NodeSerializationTest {
@Test
public void oldVersionDiscarded() throws ConnectionException {
- final Id nodeId = createId( "test" );
+ final Id nodeId = IdGenerator.createId( "test" );
final long version1 = System.currentTimeMillis();
final long version2 = version1 + 1;
@@ -171,9 +172,9 @@ public class NodeSerializationTest {
@Test
public void multiGet() throws ConnectionException {
- final Id nodeId1 = createId( "test" );
- final Id nodeId2 = createId( "test" );
- final Id nodeId3 = createId( "test" );
+ final Id nodeId1 = IdGenerator.createId( "test" );
+ final Id nodeId2 = IdGenerator.createId( "test" );
+ final Id nodeId3 = IdGenerator.createId( "test" );
final long version = System.currentTimeMillis();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/802dcdec/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/EdgeDataMigrationImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/EdgeDataMigrationImplTest.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/EdgeDataMigrationImplTest.java
new file mode 100644
index 0000000..09259f1
--- /dev/null
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/EdgeDataMigrationImplTest.java
@@ -0,0 +1,177 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence.graph.serialization.impl.migration;
+
+
+import java.util.List;
+
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+import org.apache.usergrid.persistence.core.guice.DataMigrationResetRule;
+import org.apache.usergrid.persistence.core.migration.data.DataMigrationManager;
+import org.apache.usergrid.persistence.core.migration.data.TestProgressObserver;
+import org.apache.usergrid.persistence.core.migration.data.newimpls.MigrationDataProvider;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
+import org.apache.usergrid.persistence.core.test.ITRunner;
+import org.apache.usergrid.persistence.core.test.UseModules;
+import org.apache.usergrid.persistence.core.util.IdGenerator;
+import org.apache.usergrid.persistence.graph.Edge;
+import org.apache.usergrid.persistence.graph.GraphManager;
+import org.apache.usergrid.persistence.graph.GraphManagerFactory;
+import org.apache.usergrid.persistence.graph.guice.TestGraphModule;
+import org.apache.usergrid.persistence.graph.impl.SimpleSearchEdgeType;
+import org.apache.usergrid.persistence.graph.serialization.impl.GraphDataVersions;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.google.inject.Inject;
+
+import net.jcip.annotations.NotThreadSafe;
+
+import rx.Observable;
+
+import static org.apache.usergrid.persistence.graph.test.util.EdgeTestUtils.createEdge;
+import static org.apache.usergrid.persistence.core.util.IdGenerator.createId;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+
+@NotThreadSafe
+@RunWith(ITRunner.class)
+@UseModules({ TestGraphModule.class })
+public class EdgeDataMigrationImplTest implements DataMigrationResetRule.DataMigrationManagerProvider {
+
+
+ @Inject
+ public DataMigrationManager dataMigrationManager;
+
+ @Inject
+ public GraphManagerFactory graphManagerFactory;
+
+ @Inject
+ public EdgeDataMigrationImpl edgeDataMigrationImpl;
+
+ /**
+ * Rule to do the resets we need
+ */
+ @Rule
+ public DataMigrationResetRule migrationTestRule = new DataMigrationResetRule(this, GraphMigrationPlugin.PLUGIN_NAME, GraphDataVersions.INITIAL.getVersion());
+
+
+
+
+ @Test
+ public void testIdMapping() throws Throwable {
+
+ assertEquals( "version 0 expected", GraphDataVersions.INITIAL.getVersion(),
+ dataMigrationManager.getCurrentVersion( GraphMigrationPlugin.PLUGIN_NAME ) );
+
+
+ final ApplicationScope applicationScope = new ApplicationScopeImpl( IdGenerator.createId( "application" ) );
+
+
+ GraphManager gm = graphManagerFactory.createEdgeManager(applicationScope );
+
+
+ final Id sourceId1 = IdGenerator.createId( "source1" );
+
+ final Id sourceId2 = IdGenerator.createId( "source2" );
+
+
+ final Id target1 = IdGenerator.createId( "target1" );
+
+ final Id target2 = IdGenerator.createId( "target2" );
+
+
+ Edge s1t1 = createEdge(sourceId1, "test", target1 );
+
+ Edge s1t2 = createEdge( sourceId1, "baz", target2 );
+
+ Edge s2t1 = createEdge(sourceId2, "foo", target1);
+
+ Edge s2t2 = createEdge( sourceId2, "bar", target2 );
+
+
+
+
+ gm.writeEdge( s1t1 ).toBlocking().last();
+ gm.writeEdge( s1t2 ).toBlocking().last();
+ gm.writeEdge( s2t1 ).toBlocking().last();
+ gm.writeEdge( s2t2 ).toBlocking().last();
+
+
+ //walk from s1 and s2
+ final Observable<GraphNode> graphNodes = Observable.from( new GraphNode( applicationScope, sourceId1), new GraphNode(applicationScope, sourceId2 ) );
+
+ final MigrationDataProvider<GraphNode> testMigrationProvider = new MigrationDataProvider<GraphNode>() {
+ @Override
+ public Observable<GraphNode> getData() {
+ return graphNodes;
+ }
+ };
+
+
+ final TestProgressObserver progressObserver = new TestProgressObserver();
+
+
+
+ //read everything in previous version format and put it into our types.
+
+
+ final int returned = edgeDataMigrationImpl.migrate( GraphDataVersions.INITIAL.getVersion() , testMigrationProvider, progressObserver );
+ //perform the migration
+
+ assertEquals( "Correct version returned", returned, GraphDataVersions.META_SHARDING.getVersion() );
+ assertFalse( "Progress observer should not have failed", progressObserver.getFailed() );
+ assertTrue( "Progress observer should have update messages", progressObserver.getUpdates().size() > 0 );
+
+
+ //now check we can still read our data.
+
+ List<String> source1Edges =
+ gm.getEdgeTypesFromSource( new SimpleSearchEdgeType( sourceId1, null, null ) ).toList().toBlocking().last();
+
+ //now check both edge types are present
+
+ assertTrue( "Edge type present", source1Edges.contains( "test" ) );
+ assertTrue( "Edge type present", source1Edges.contains( "baz" ) );
+
+
+ List<String> source2Edges =
+ gm.getEdgeTypesFromSource( new SimpleSearchEdgeType( sourceId2, null, null ) ).toList().toBlocking().last();
+
+ //now check both edge types are present
+
+ assertTrue( "Edge type present", source2Edges.contains( "foo" ) );
+ assertTrue( "Edge type present", source2Edges.contains( "bar" ) );
+
+
+
+ }
+
+
+ @Override
+ public DataMigrationManager getDataMigrationManager() {
+ return dataMigrationManager;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/802dcdec/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/EdgeShardSerializationTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/EdgeShardSerializationTest.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/EdgeShardSerializationTest.java
index 9a87bba..2641ed7 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/EdgeShardSerializationTest.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/EdgeShardSerializationTest.java
@@ -31,6 +31,7 @@ import org.apache.usergrid.persistence.core.guice.MigrationManagerRule;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.test.ITRunner;
import org.apache.usergrid.persistence.core.test.UseModules;
+import org.apache.usergrid.persistence.core.util.IdGenerator;
import org.apache.usergrid.persistence.graph.guice.TestGraphModule;
import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
@@ -40,7 +41,7 @@ import com.google.inject.Inject;
import com.netflix.astyanax.MutationBatch;
import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
-import static org.apache.usergrid.persistence.graph.test.util.EdgeTestUtils.createId;
+import static org.apache.usergrid.persistence.core.util.IdGenerator.createId;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.mockito.Mockito.mock;
@@ -79,7 +80,7 @@ public class EdgeShardSerializationTest {
@Test
public void saveReturn() throws ConnectionException {
- final Id now = createId( "test" );
+ final Id now = IdGenerator.createId( "test" );
final long timestamp = 10000l;
@@ -137,7 +138,7 @@ public class EdgeShardSerializationTest {
@Test
public void saveReturnDelete() throws ConnectionException {
- final Id now = createId( "test" );
+ final Id now = IdGenerator.createId( "test" );
final long timestamp = 10000l;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/802dcdec/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/NodeShardAllocationTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/NodeShardAllocationTest.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/NodeShardAllocationTest.java
index 4e572c0..ac965cd 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/NodeShardAllocationTest.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/NodeShardAllocationTest.java
@@ -32,6 +32,7 @@ import org.mockito.ArgumentCaptor;
import org.apache.usergrid.persistence.core.consistency.TimeService;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.core.util.IdGenerator;
import org.apache.usergrid.persistence.graph.GraphFig;
import org.apache.usergrid.persistence.graph.MarkedEdge;
import org.apache.usergrid.persistence.graph.SearchByIdType;
@@ -46,7 +47,7 @@ import com.netflix.astyanax.MutationBatch;
import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
import static junit.framework.TestCase.assertTrue;
-import static org.apache.usergrid.persistence.graph.test.util.EdgeTestUtils.createId;
+import static org.apache.usergrid.persistence.core.util.IdGenerator.createId;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.fail;
@@ -140,7 +141,7 @@ public class NodeShardAllocationTest {
new NodeShardAllocationImpl( edgeShardSerialization, edgeColumnFamilies, shardedEdgeSerialization,
nodeShardCounterSerialization, timeService, graphFig, shardGroupCompaction );
- final Id nodeId = createId( "test" );
+ final Id nodeId = IdGenerator.createId( "test" );
final String type = "type";
final String subType = "subType";
@@ -183,7 +184,7 @@ public class NodeShardAllocationTest {
new NodeShardAllocationImpl( edgeShardSerialization, edgeColumnFamilies, shardedEdgeSerialization,
nodeShardApproximation, timeService, graphFig, shardGroupCompaction );
- final Id nodeId = createId( "test" );
+ final Id nodeId = IdGenerator.createId( "test" );
final String type = "type";
final String subType = "subType";
@@ -231,7 +232,7 @@ public class NodeShardAllocationTest {
new NodeShardAllocationImpl( edgeShardSerialization, edgeColumnFamilies, shardedEdgeSerialization,
nodeShardApproximation, timeService, graphFig, shardGroupCompaction );
- final Id nodeId = createId( "test" );
+ final Id nodeId = IdGenerator.createId( "test" );
final String type = "type";
final String subType = "subType";
@@ -266,8 +267,8 @@ public class NodeShardAllocationTest {
* Just use 2 edges. It means that we won't generate a boatload of data and kill our test. We just want
* to check that the one we want to return is correct
*/
- SimpleMarkedEdge skipped = new SimpleMarkedEdge( nodeId, type, createId( subType ), 10000, false );
- SimpleMarkedEdge keep = new SimpleMarkedEdge( nodeId, type, createId( subType ), 20000, false );
+ SimpleMarkedEdge skipped = new SimpleMarkedEdge( nodeId, type, IdGenerator.createId( subType ), 10000, false );
+ SimpleMarkedEdge keep = new SimpleMarkedEdge( nodeId, type, IdGenerator.createId( subType ), 20000, false );
//allocate some extra to ensure we seek the right value
List<MarkedEdge> edges = new ArrayList( numToIterate + 100 );
@@ -346,7 +347,7 @@ public class NodeShardAllocationTest {
new NodeShardAllocationImpl( edgeShardSerialization, edgeColumnFamilies, shardedEdgeSerialization,
nodeShardApproximation, timeService, graphFig, shardGroupCompaction );
- final Id nodeId = createId( "test" );
+ final Id nodeId = IdGenerator.createId( "test" );
final String type = "type";
final String subType = "subType";
@@ -365,9 +366,9 @@ public class NodeShardAllocationTest {
final long shardCount = graphFig.getShardSize();
- final SimpleMarkedEdge skippedEdge = new SimpleMarkedEdge( nodeId, type, createId( "subType" ), 10000l, false );
+ final SimpleMarkedEdge skippedEdge = new SimpleMarkedEdge( nodeId, type, IdGenerator.createId( "subType" ), 10000l, false );
final SimpleMarkedEdge returnedEdge =
- new SimpleMarkedEdge( nodeId, type, createId( "subType" ), 10005l, false );
+ new SimpleMarkedEdge( nodeId, type, IdGenerator.createId( "subType" ), 10005l, false );
List<MarkedEdge> iteratedEdges = new ArrayList<>( ( int ) shardCount );
@@ -437,7 +438,7 @@ public class NodeShardAllocationTest {
new NodeShardAllocationImpl( edgeShardSerialization, edgeColumnFamilies, shardedEdgeSerialization,
nodeShardApproximation, timeService, graphFig, shardGroupCompaction );
- final Id nodeId = createId( "test" );
+ final Id nodeId = IdGenerator.createId( "test" );
final String type = "type";
final String subType = "subType";
@@ -467,7 +468,7 @@ public class NodeShardAllocationTest {
final SimpleMarkedEdge returnedEdge =
- new SimpleMarkedEdge( nodeId, type, createId( "subType" ), 10005l, false );
+ new SimpleMarkedEdge( nodeId, type, IdGenerator.createId( "subType" ), 10005l, false );
final Iterator<MarkedEdge> edgeIterator = Collections.singleton( ( MarkedEdge ) returnedEdge ).iterator();
@@ -504,7 +505,7 @@ public class NodeShardAllocationTest {
new NodeShardAllocationImpl( edgeShardSerialization, edgeColumnFamilies, shardedEdgeSerialization,
nodeShardApproximation, timeService, graphFig, shardGroupCompaction );
- final Id nodeId = createId( "test" );
+ final Id nodeId = IdGenerator.createId( "test" );
final String type = "type";
final String subType = "subType";
@@ -644,7 +645,7 @@ public class NodeShardAllocationTest {
new NodeShardAllocationImpl( edgeShardSerialization, edgeColumnFamilies, shardedEdgeSerialization,
nodeShardApproximation, timeService, graphFig, shardGroupCompaction );
- final Id nodeId = createId( "test" );
+ final Id nodeId = IdGenerator.createId( "test" );
final String type = "type";
final String subType = "subType";
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/802dcdec/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/NodeShardCacheTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/NodeShardCacheTest.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/NodeShardCacheTest.java
index c9f87fd..3b717a4 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/NodeShardCacheTest.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/NodeShardCacheTest.java
@@ -29,6 +29,7 @@ import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.core.util.IdGenerator;
import org.apache.usergrid.persistence.graph.GraphFig;
import org.apache.usergrid.persistence.graph.serialization.impl.shard.impl.NodeShardCacheImpl;
import org.apache.usergrid.persistence.model.entity.Id;
@@ -37,7 +38,7 @@ import org.apache.usergrid.persistence.model.util.UUIDGenerator;
import com.google.common.base.Optional;
import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
-import static org.apache.usergrid.persistence.graph.test.util.EdgeTestUtils.createId;
+import static org.apache.usergrid.persistence.core.util.IdGenerator.createId;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertSame;
import static org.junit.Assert.assertTrue;
@@ -76,7 +77,7 @@ public class NodeShardCacheTest {
final NodeShardAllocation allocation = mock( NodeShardAllocation.class );
- final Id id = createId( "test" );
+ final Id id = IdGenerator.createId( "test" );
final String edgeType = "edge";
@@ -151,7 +152,7 @@ public class NodeShardCacheTest {
final NodeShardAllocation allocation = mock( NodeShardAllocation.class );
- final Id id = createId( "test" );
+ final Id id = IdGenerator.createId( "test" );
final String edgeType = "edge";
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/802dcdec/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/ShardGroupCompactionTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/ShardGroupCompactionTest.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/ShardGroupCompactionTest.java
index 71b1f0d..68b6f8f 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/ShardGroupCompactionTest.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/ShardGroupCompactionTest.java
@@ -33,12 +33,13 @@ import org.apache.usergrid.persistence.core.consistency.TimeService;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
import org.apache.usergrid.persistence.core.task.TaskExecutor;
+import org.apache.usergrid.persistence.core.util.IdGenerator;
import org.apache.usergrid.persistence.graph.GraphFig;
import org.apache.usergrid.persistence.graph.serialization.impl.shard.impl.ShardGroupCompactionImpl;
import com.netflix.astyanax.Keyspace;
-import static org.apache.usergrid.persistence.graph.test.util.EdgeTestUtils.createId;
+import static org.apache.usergrid.persistence.core.util.IdGenerator.createId;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
import static org.mockito.Mockito.mock;
@@ -59,7 +60,7 @@ public class ShardGroupCompactionTest {
when( graphFig.getShardAuditWorkerQueueSize() ).thenReturn( 1000 );
- this.scope = new ApplicationScopeImpl( createId( "application" ) );
+ this.scope = new ApplicationScopeImpl( IdGenerator.createId( "application" ) );
}
@@ -99,7 +100,7 @@ public class ShardGroupCompactionTest {
edgeColumnFamilies, keyspace, edgeShardSerialization, taskExecutor );
- DirectedEdgeMeta directedEdgeMeta = DirectedEdgeMeta.fromSourceNode( createId( "source" ), "test" );
+ DirectedEdgeMeta directedEdgeMeta = DirectedEdgeMeta.fromSourceNode( IdGenerator.createId( "source" ), "test" );
try {
compaction.compact( this.scope, directedEdgeMeta, group );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/802dcdec/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/count/NodeShardApproximationTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/count/NodeShardApproximationTest.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/count/NodeShardApproximationTest.java
index e08abd5..32a0cda 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/count/NodeShardApproximationTest.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/count/NodeShardApproximationTest.java
@@ -46,6 +46,7 @@ import org.slf4j.LoggerFactory;
import org.apache.usergrid.persistence.core.astyanax.MultiTennantColumnFamilyDefinition;
import org.apache.usergrid.persistence.core.consistency.TimeService;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.core.util.IdGenerator;
import org.apache.usergrid.persistence.graph.GraphFig;
import org.apache.usergrid.persistence.graph.serialization.impl.shard.DirectedEdgeMeta;
import org.apache.usergrid.persistence.graph.serialization.impl.shard.NodeShardApproximation;
@@ -64,7 +65,7 @@ import com.netflix.astyanax.model.ColumnFamily;
import com.netflix.astyanax.model.ConsistencyLevel;
import com.netflix.astyanax.retry.RetryPolicy;
-import static org.apache.usergrid.persistence.graph.test.util.EdgeTestUtils.createId;
+import static org.apache.usergrid.persistence.core.util.IdGenerator.createId;
import static org.junit.Assert.assertEquals;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.mock;
@@ -120,7 +121,7 @@ public class NodeShardApproximationTest {
new NodeShardApproximationImpl( graphFig, nodeShardCounterSerialization, timeService );
- final Id id = createId( "test" );
+ final Id id = IdGenerator.createId( "test" );
final Shard shard = new Shard( 0, 0, true );
final String type = "type";
final String type2 = "subType";
@@ -149,7 +150,7 @@ public class NodeShardApproximationTest {
final int increments = 1000000;
final int workers = Runtime.getRuntime().availableProcessors() * 2;
- final Id id = createId( "test" );
+ final Id id = IdGenerator.createId( "test" );
final String type = "type";
final String type2 = "subType";
@@ -216,7 +217,7 @@ public class NodeShardApproximationTest {
final int increments = 1000000;
final int workers = Runtime.getRuntime().availableProcessors() * 2;
- final Id id = createId( "test" );
+ final Id id = IdGenerator.createId( "test" );
final String type = "type";
final String type2 = "subType";
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/802dcdec/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/count/NodeShardCounterSerializationTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/count/NodeShardCounterSerializationTest.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/count/NodeShardCounterSerializationTest.java
index f80f44d..86f2491 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/count/NodeShardCounterSerializationTest.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/count/NodeShardCounterSerializationTest.java
@@ -20,7 +20,6 @@ package org.apache.usergrid.persistence.graph.serialization.impl.shard.count;
import org.junit.Before;
-import org.junit.ClassRule;
import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -31,6 +30,7 @@ import org.apache.usergrid.persistence.core.guice.MigrationManagerRule;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.test.ITRunner;
import org.apache.usergrid.persistence.core.test.UseModules;
+import org.apache.usergrid.persistence.core.util.IdGenerator;
import org.apache.usergrid.persistence.graph.GraphFig;
import org.apache.usergrid.persistence.graph.guice.TestGraphModule;
import org.apache.usergrid.persistence.graph.serialization.EdgeSerialization;
@@ -43,7 +43,7 @@ import com.google.inject.Inject;
import com.netflix.astyanax.Keyspace;
import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
-import static org.apache.usergrid.persistence.graph.test.util.EdgeTestUtils.createId;
+import static org.apache.usergrid.persistence.core.util.IdGenerator.createId;
import static org.junit.Assert.assertEquals;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -92,7 +92,7 @@ public class NodeShardCounterSerializationTest {
public void testWritesRead() throws ConnectionException {
- final Id id = createId( "test" );
+ final Id id = IdGenerator.createId( "test" );
ShardKey key1 = new ShardKey( scope, new Shard( 0, 0, false ), DirectedEdgeMeta.fromSourceNode( id, "type1" ) );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/802dcdec/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardEntryGroupIteratorTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardEntryGroupIteratorTest.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardEntryGroupIteratorTest.java
index 9544865..0eccc3e 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardEntryGroupIteratorTest.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardEntryGroupIteratorTest.java
@@ -31,13 +31,14 @@ import org.junit.Test;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
+import org.apache.usergrid.persistence.core.util.IdGenerator;
import org.apache.usergrid.persistence.graph.serialization.impl.shard.DirectedEdgeMeta;
import org.apache.usergrid.persistence.graph.serialization.impl.shard.Shard;
import org.apache.usergrid.persistence.graph.serialization.impl.shard.ShardEntryGroup;
import org.apache.usergrid.persistence.graph.serialization.impl.shard.ShardGroupCompaction;
import static junit.framework.TestCase.assertTrue;
-import static org.apache.usergrid.persistence.graph.test.util.EdgeTestUtils.createId;
+import static org.apache.usergrid.persistence.core.util.IdGenerator.createId;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.mockito.Matchers.eq;
@@ -52,8 +53,8 @@ public class ShardEntryGroupIteratorTest {
@Test(expected = IllegalArgumentException.class)
public void noShards() {
- final ApplicationScope scope = new ApplicationScopeImpl( createId( "application" ) );
- final DirectedEdgeMeta directedEdgeMeta = DirectedEdgeMeta.fromSourceNode( createId( "source" ), "test" );
+ final ApplicationScope scope = new ApplicationScopeImpl( IdGenerator.createId( "application" ) );
+ final DirectedEdgeMeta directedEdgeMeta = DirectedEdgeMeta.fromSourceNode( IdGenerator.createId( "source" ), "test" );
final ShardGroupCompaction shardGroupCompaction = mock( ShardGroupCompaction.class );
final long delta = 10000;
final Iterator<Shard> noShards = Collections.<Shard>emptyList().iterator();
@@ -66,8 +67,8 @@ public class ShardEntryGroupIteratorTest {
@Test
public void existingSingleShard() {
- final ApplicationScope scope = new ApplicationScopeImpl( createId( "application" ) );
- final DirectedEdgeMeta directedEdgeMeta = DirectedEdgeMeta.fromSourceNode( createId( "source" ), "test" );
+ final ApplicationScope scope = new ApplicationScopeImpl( IdGenerator.createId( "application" ) );
+ final DirectedEdgeMeta directedEdgeMeta = DirectedEdgeMeta.fromSourceNode( IdGenerator.createId( "source" ), "test" );
final ShardGroupCompaction shardGroupCompaction = mock( ShardGroupCompaction.class );
@@ -119,8 +120,8 @@ public class ShardEntryGroupIteratorTest {
@Test
public void boundedShardSets() {
- final ApplicationScope scope = new ApplicationScopeImpl( createId( "application" ) );
- final DirectedEdgeMeta directedEdgeMeta = DirectedEdgeMeta.fromSourceNode( createId( "source" ), "test" );
+ final ApplicationScope scope = new ApplicationScopeImpl( IdGenerator.createId( "application" ) );
+ final DirectedEdgeMeta directedEdgeMeta = DirectedEdgeMeta.fromSourceNode( IdGenerator.createId( "source" ), "test" );
final ShardGroupCompaction shardGroupCompaction = mock( ShardGroupCompaction.class );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/802dcdec/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/comparators/SourceDirectedEdgeDescendingComparatorTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/comparators/SourceDirectedEdgeDescendingComparatorTest.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/comparators/SourceDirectedEdgeDescendingComparatorTest.java
index bb9fd0d..1e18c67 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/comparators/SourceDirectedEdgeDescendingComparatorTest.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/comparators/SourceDirectedEdgeDescendingComparatorTest.java
@@ -26,11 +26,12 @@ import java.util.UUID;
import org.junit.Test;
+import org.apache.usergrid.persistence.core.util.IdGenerator;
import org.apache.usergrid.persistence.graph.impl.SimpleMarkedEdge;
import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
-import static org.apache.usergrid.persistence.graph.test.util.EdgeTestUtils.createId;
+import static org.apache.usergrid.persistence.core.util.IdGenerator.createId;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@@ -43,8 +44,8 @@ public class SourceDirectedEdgeDescendingComparatorTest {
@Test
public void sameEdges() {
- final Id sourceId = createId( "source" );
- final Id targetId = createId( "target" );
+ final Id sourceId = IdGenerator.createId( "source" );
+ final Id targetId = IdGenerator.createId( "target" );
final String type = "type";
final long timestamp = 10000;
@@ -65,8 +66,8 @@ public class SourceDirectedEdgeDescendingComparatorTest {
@Test
public void timestampDifferent() {
- final Id sourceId = createId( "source" );
- final Id targetId = createId( "target" );
+ final Id sourceId = IdGenerator.createId( "source" );
+ final Id targetId = IdGenerator.createId( "target" );
final String type = "type";
final long timestamp = 10000;
@@ -88,9 +89,9 @@ public class SourceDirectedEdgeDescendingComparatorTest {
@Test
public void uuidDifferent() {
- final Id sourceId1 = createId( "source" );
- final Id sourceId2 = createId( "source" );
- final Id targetId = createId( "target" );
+ final Id sourceId1 = IdGenerator.createId( "source" );
+ final Id sourceId2 = IdGenerator.createId( "source" );
+ final Id targetId = IdGenerator.createId( "target" );
final String type = "type";
final long timestamp = 10000;
@@ -114,9 +115,9 @@ public class SourceDirectedEdgeDescendingComparatorTest {
final UUID sourceId = UUIDGenerator.newTimeUUID();
- final Id sourceId1 = createId( sourceId, "source1" );
- final Id sourceId2 = createId( sourceId, "source2" );
- final Id targetId = createId( "target" );
+ final Id sourceId1 = IdGenerator.createId( sourceId, "source1" );
+ final Id sourceId2 = IdGenerator.createId( sourceId, "source2" );
+ final Id targetId = IdGenerator.createId( "target" );
final String type = "type";
final long timestamp = 10000;
[19/50] [abbrv] incubator-usergrid git commit: Merge branch
'two-dot-o' into USERGRID-405
Posted by sf...@apache.org.
Merge branch 'two-dot-o' into USERGRID-405
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/49fe0ac8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/49fe0ac8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/49fe0ac8
Branch: refs/heads/USERGRID-480
Commit: 49fe0ac86f5d0b17ca0f9c18b3f6e701505e0612
Parents: e54b263 ef3f77d
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Mar 4 17:45:28 2015 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Mar 4 17:45:28 2015 -0700
----------------------------------------------------------------------
.../java/org/apache/usergrid/persistence/index/query/Query.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
[03/50] [abbrv] incubator-usergrid git commit: WIP,
still needs refactored and cleaned up.
Posted by sf...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d3f8ee61/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/ProgressObserver.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/ProgressObserver.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/ProgressObserver.java
new file mode 100644
index 0000000..36c721a
--- /dev/null
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/ProgressObserver.java
@@ -0,0 +1,50 @@
+/*
+ *
+ * *
+ * * Licensed to the Apache Software Foundation (ASF) under one
+ * * or more contributor license agreements. See the NOTICE file
+ * * distributed with this work for additional information
+ * * regarding copyright ownership. The ASF licenses this file
+ * * to you under the Apache License, Version 2.0 (the
+ * * "License"); you may not use this file except in compliance
+ * * with the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing,
+ * * software distributed under the License is distributed on an
+ * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * * KIND, either express or implied. See the License for the
+ * * specific language governing permissions and limitations
+ * * under the License.
+ * *
+ *
+ */
+
+package org.apache.usergrid.persistence.core.migration.data.newimpls;
+
+
+public interface ProgressObserver{
+ /**
+ * Mark the migration as failed
+ * @param migrationVersion The migration version running during the failure
+ * @param reason The reason to save
+ */
+ public void failed(final int migrationVersion, final String reason);
+
+ /**
+ * Mark the migration as failed with a stack trace
+ * @param migrationVersion The migration version running during the failure
+ * @param reason The error description to save
+ * @param throwable The error that happened
+ */
+ public void failed(final int migrationVersion, final String reason, final Throwable throwable);
+
+
+ /**
+ * Update the status of the migration with the message
+ *
+ * @param message The message to save for the status
+ */
+ public void update(final int migrationVersion, final String message);
+ }
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d3f8ee61/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/VersionedData.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/VersionedData.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/VersionedData.java
new file mode 100644
index 0000000..14d488b
--- /dev/null
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/VersionedData.java
@@ -0,0 +1,38 @@
+/*
+ *
+ * *
+ * * Licensed to the Apache Software Foundation (ASF) under one
+ * * or more contributor license agreements. See the NOTICE file
+ * * distributed with this work for additional information
+ * * regarding copyright ownership. The ASF licenses this file
+ * * to you under the Apache License, Version 2.0 (the
+ * * "License"); you may not use this file except in compliance
+ * * with the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing,
+ * * software distributed under the License is distributed on an
+ * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * * KIND, either express or implied. See the License for the
+ * * specific language governing permissions and limitations
+ * * under the License.
+ * *
+ *
+ */
+
+package org.apache.usergrid.persistence.core.migration.data.newimpls;
+
+
+/**
+ * Marker interface for implementations that may migrate their internal format
+ */
+public interface VersionedData {
+
+ /**
+ * Get the version of the implementation
+ * @return
+ */
+ public int getImplementationVersion();
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d3f8ee61/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/VersionedSet.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/VersionedSet.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/VersionedSet.java
new file mode 100644
index 0000000..a5123cf
--- /dev/null
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/VersionedSet.java
@@ -0,0 +1,113 @@
+/*
+ *
+ * *
+ * * Licensed to the Apache Software Foundation (ASF) under one
+ * * or more contributor license agreements. See the NOTICE file
+ * * distributed with this work for additional information
+ * * regarding copyright ownership. The ASF licenses this file
+ * * to you under the Apache License, Version 2.0 (the
+ * * "License"); you may not use this file except in compliance
+ * * with the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing,
+ * * software distributed under the License is distributed on an
+ * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * * KIND, either express or implied. See the License for the
+ * * specific language governing permissions and limitations
+ * * under the License.
+ * *
+ *
+ */
+
+package org.apache.usergrid.persistence.core.migration.data.newimpls;
+
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+
+import com.google.inject.Inject;
+
+
+public class VersionedSet<T extends VersionedData> {
+
+
+ /**
+ * Cache so that after our initial lookup, it O(1) since this will be used heavily
+ *
+ */
+ private Map<Integer, MigrationRelationship<T>> cacheVersion = new HashMap<>();
+
+ private List<T> orderedVersions = new ArrayList<>();
+
+
+
+
+ public VersionedSet(final Collection<T> versions){
+
+ orderedVersions.addAll(versions );
+ Collections.sort( orderedVersions, new VersionedDataComparator() );
+
+ }
+
+
+ /**
+ * Get the migration relationship based on our current version
+ * @param currentVersion
+ * @return
+ */
+ public MigrationRelationship<T> getCurrentReadVersion(final int currentVersion){
+
+ final MigrationRelationship<T> relationship = cacheVersion.get( currentVersion );
+
+ if(relationship != null){
+ return relationship;
+ }
+
+ //not there, find it. Not the most efficient, but it happens once per version, which rarely changes, so not a big deal
+
+
+ for(T current: orderedVersions){
+ //not our instance
+ if(current.getImplementationVersion() > currentVersion){
+ continue;
+ }
+
+
+ //we always go from our first match to our highest version. Any versions between can be skipped
+ final MigrationRelationship<T> migrationRelationship = new MigrationRelationship<>( current, orderedVersions.get( 0 ) );
+
+ cacheVersion.put( currentVersion, migrationRelationship );
+
+ return migrationRelationship;
+
+ }
+
+ //if we get here, something is wrong
+ throw new IllegalArgumentException( "Could not find a migration version for version " + currentVersion + " min found was " + orderedVersions.get( orderedVersions.size()-1 ) );
+
+
+ }
+
+
+ /**
+ * Orders from high to low
+ */
+ private static final class VersionedDataComparator implements Comparator<VersionedData>
+ {
+
+ @Override
+ public int compare( final VersionedData o1, final VersionedData o2 ) {
+ return Integer.compare( o1.getImplementationVersion(), o2.getImplementationVersion())*-1;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d3f8ee61/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/schema/MigrationStrategy.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/schema/MigrationStrategy.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/schema/MigrationStrategy.java
deleted file mode 100644
index e1714d0..0000000
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/schema/MigrationStrategy.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- *
- * * Licensed to the Apache Software Foundation (ASF) under one or more
- * * contributor license agreements. The ASF licenses this file to You
- * * under the Apache License, Version 2.0 (the "License"); you may not
- * * use this file except in compliance with the License.
- * * You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing, software
- * * distributed under the License is distributed on an "AS IS" BASIS,
- * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * * See the License for the specific language governing permissions and
- * * limitations under the License. For additional information regarding
- * * copyright in this work, please see the NOTICE file in the top level
- * * directory of this distribution.
- *
- */
-package org.apache.usergrid.persistence.core.migration.schema;
-
-import org.apache.usergrid.persistence.core.scope.ApplicationEntityGroup;
-import org.apache.usergrid.persistence.core.migration.data.DataMigration;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.model.entity.Id;
-import rx.Observable;
-import rx.functions.Func1;
-
-
-/**
- * Interface to encapsulate directional migrations
- */
-public interface MigrationStrategy<T> {
- /**
- * Returns the migration pattern to use
- * @return
- */
- public MigrationRelationship<T> getMigration();
-
- public int getVersion();
-
- public class MigrationRelationship<T> {
- private final T from;
- private final T to;
-
- public MigrationRelationship(T from,T to){
- this.from = from;
- this.to = to;
- }
- public T from(){
- return from;
- }
-
- public T to(){
- return to;
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d3f8ee61/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/rx/AllEntitiesInSystemObservable.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/rx/AllEntitiesInSystemObservable.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/rx/AllEntitiesInSystemObservable.java
index e029f92..e7a7aca 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/rx/AllEntitiesInSystemObservable.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/rx/AllEntitiesInSystemObservable.java
@@ -21,7 +21,7 @@ package org.apache.usergrid.persistence.core.rx;
import org.apache.usergrid.persistence.core.scope.ApplicationEntityGroup;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.model.entity.Id;
+
import rx.Observable;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d3f8ee61/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/scope/ApplicationEntityGroup.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/scope/ApplicationEntityGroup.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/scope/ApplicationEntityGroup.java
deleted file mode 100644
index d0d81c7..0000000
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/scope/ApplicationEntityGroup.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- *
- * * Licensed to the Apache Software Foundation (ASF) under one or more
- * * contributor license agreements. The ASF licenses this file to You
- * * under the Apache License, Version 2.0 (the "License"); you may not
- * * use this file except in compliance with the License.
- * * You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing, software
- * * distributed under the License is distributed on an "AS IS" BASIS,
- * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * * See the License for the specific language governing permissions and
- * * limitations under the License. For additional information regarding
- * * copyright in this work, please see the NOTICE file in the top level
- * * directory of this distribution.
- *
- */
-package org.apache.usergrid.persistence.core.scope;
-
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.model.entity.Id;
-
-import java.util.List;
-
-/**
- * Get the entity data. Immutable bean for fast access
- */
-public final class ApplicationEntityGroup<T extends ApplicationScope> {
- public final T applicationScope;
- public final List<EntityIdScope<T>> entityIds;
-
- public ApplicationEntityGroup(final T applicationScope, final List<EntityIdScope<T>> entityIds) {
- this.applicationScope = applicationScope;
- this.entityIds = entityIds;
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d3f8ee61/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/scope/EntityIdScope.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/scope/EntityIdScope.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/scope/EntityIdScope.java
deleted file mode 100644
index 73b47fd..0000000
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/scope/EntityIdScope.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- *
- * * Licensed to the Apache Software Foundation (ASF) under one or more
- * * contributor license agreements. The ASF licenses this file to You
- * * under the Apache License, Version 2.0 (the "License"); you may not
- * * use this file except in compliance with the License.
- * * You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing, software
- * * distributed under the License is distributed on an "AS IS" BASIS,
- * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * * See the License for the specific language governing permissions and
- * * limitations under the License. For additional information regarding
- * * copyright in this work, please see the NOTICE file in the top level
- * * directory of this distribution.
- *
- */
-package org.apache.usergrid.persistence.core.scope;
-
-import org.apache.usergrid.persistence.model.entity.Id;
-
-/**
- * Tuple containing collectionscope and entityid
- */
-public class EntityIdScope<T extends ApplicationScope>{
- private final Id id;
- private final T collectionScope;
-
- public EntityIdScope(Id id, T collectionScope){
- this.id = id;
- this.collectionScope = collectionScope;
- }
-
-
- public Id getId() {
- return id;
- }
-
- public T getCollectionScope() {
- return collectionScope;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d3f8ee61/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/guice/MaxMigrationVersion.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/guice/MaxMigrationVersion.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/guice/MaxMigrationVersion.java
index d3b9b49..58cf5a6 100644
--- a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/guice/MaxMigrationVersion.java
+++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/guice/MaxMigrationVersion.java
@@ -21,7 +21,6 @@ package org.apache.usergrid.persistence.core.guice;
import org.apache.usergrid.persistence.core.migration.data.ApplicationDataMigration;
-import org.apache.usergrid.persistence.core.migration.data.DataMigration;
import org.apache.usergrid.persistence.core.scope.ApplicationEntityGroup;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import rx.Observable;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d3f8ee61/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/guice/TestModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/guice/TestModule.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/guice/TestModule.java
index 98b057b..3aeb629 100644
--- a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/guice/TestModule.java
+++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/guice/TestModule.java
@@ -24,9 +24,8 @@ import java.io.IOException;
import com.google.inject.AbstractModule;
import com.netflix.config.ConfigurationManager;
-import org.apache.usergrid.persistence.core.rx.AllEntitiesInSystemObservable;
+
import org.apache.usergrid.persistence.core.scope.ApplicationEntityGroup;
-import rx.Observable;
public abstract class TestModule extends AbstractModule {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d3f8ee61/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManagerImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManagerImplTest.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManagerImplTest.java
index e067f60..20972f6 100644
--- a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManagerImplTest.java
+++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManagerImplTest.java
@@ -33,7 +33,6 @@ import org.apache.usergrid.persistence.core.scope.EntityIdScope;
import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.entity.SimpleId;
import org.junit.Test;
-import org.mockito.Mockito;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
@@ -42,7 +41,6 @@ import rx.Observable;
import rx.Subscriber;
import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.anyInt;
import static org.mockito.Mockito.*;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d3f8ee61/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeDataMigrationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeDataMigrationImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeDataMigrationImpl.java
index 7d4fb2e..785e341 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeDataMigrationImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeDataMigrationImpl.java
@@ -23,9 +23,8 @@ import com.google.inject.Inject;
import com.netflix.astyanax.Keyspace;
import com.netflix.astyanax.MutationBatch;
import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
-import org.apache.usergrid.persistence.core.migration.data.ApplicationDataMigration;
-import org.apache.usergrid.persistence.core.migration.data.DataMigration;
-import org.apache.usergrid.persistence.core.scope.ApplicationEntityGroup;
+import org.apache.usergrid.persistence.core.migration.data.newimpls.DataMigration2;
+import org.apache.usergrid.persistence.core.migration.data.newimpls.MigrationDataProvider;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.graph.Edge;
import org.apache.usergrid.persistence.graph.GraphManager;
@@ -45,7 +44,7 @@ import java.util.concurrent.atomic.AtomicLong;
* Encapsulates data mi
*/
-public class EdgeDataMigrationImpl implements ApplicationDataMigration {
+public class EdgeDataMigrationImpl implements DataMigration2<ApplicationScope> {
private static final Logger logger = LoggerFactory.getLogger(EdgeDataMigrationImpl.class);
@@ -68,67 +67,69 @@ public class EdgeDataMigrationImpl implements ApplicationDataMigration {
}
+
+
+
@Override
- public Observable migrate(final Observable<ApplicationScope> scopes,
- final DataMigration.ProgressObserver observer) {
+ public void migrate( final MigrationDataProvider<ApplicationScope> migrationDataProvider,
+ final ProgressObserver observer ) {
final AtomicLong counter = new AtomicLong();
- return scopes.flatMap(new Func1<ApplicationScope, Observable<?>>() {
- @Override
- public Observable call(final ApplicationScope applicationScope) {
- final GraphManager gm = graphManagerFactory.createEdgeManager(applicationScope);
- final Observable<Edge> edgesFromSource = edgesFromSourceObservable.edgesFromSource(gm, applicationScope.getApplication());
- logger.info("Migrating edges scope {}", applicationScope);
-
- //get each edge from this node as a source
- return edgesFromSource
-
- //for each edge, re-index it in v2 every 1000 edges or less
- .buffer(1000)
- .doOnNext(new Action1<List<Edge>>() {
- @Override
- public void call(List<Edge> edges) {
- final MutationBatch batch =
- keyspace.prepareMutationBatch();
-
- for (Edge edge : edges) {
- logger.info("Migrating meta for edge {}", edge);
- final MutationBatch edgeBatch = edgeMigrationStrategy.getMigration().to()
- .writeEdge(applicationScope,
- edge);
- batch.mergeShallow(edgeBatch);
- }
-
- try {
- batch.execute();
- } catch (ConnectionException e) {
- throw new RuntimeException(
- "Unable to perform migration", e);
- }
-
- //update the observer so the admin can see it
- final long newCount =
- counter.addAndGet(edges.size());
-
- observer.update(getVersion(), String.format(
- "Currently running. Rewritten %d edge types",
- newCount));
- }
- });
- }
- });
+ migrationDataProvider.getData().flatMap(new Func1<ApplicationScope, Observable<?>>() {
+ @Override
+ public Observable call(final ApplicationScope applicationScope) {
+ final GraphManager gm = graphManagerFactory.createEdgeManager(applicationScope);
+ final Observable<Edge> edgesFromSource = edgesFromSourceObservable.edgesFromSource(gm, applicationScope.getApplication());
+ logger.info("Migrating edges scope {}", applicationScope);
+
+ //get each edge from this node as a source
+ return edgesFromSource
+
+ //for each edge, re-index it in v2 every 1000 edges or less
+ .buffer( 1000 )
+ //do the writes of 1k in parallel
+ .parallel( new Func1<Observable<List<Edge>>, Observable>() {
+ @Override
+ public Observable call( final Observable<List<Edge>> listObservable ) {
+ return listObservable.doOnNext( new Action1<List<Edge>>() {
+ @Override
+ public void call( List<Edge> edges ) {
+ final MutationBatch batch = keyspace.prepareMutationBatch();
+
+ for ( Edge edge : edges ) {
+ logger.info( "Migrating meta for edge {}", edge );
+ final MutationBatch edgeBatch =
+ edgeMigrationStrategy.getMigration().to()
+ .writeEdge( applicationScope, edge );
+ batch.mergeShallow( edgeBatch );
+ }
+
+ try {
+ batch.execute();
+ }
+ catch ( ConnectionException e ) {
+ throw new RuntimeException( "Unable to perform migration", e );
+ }
+
+ //update the observer so the admin can see it
+ final long newCount = counter.addAndGet( edges.size() );
+
+ observer.update( getVersion(),
+ String.format( "Currently running. Rewritten %d edge types",
+ newCount ) );
+ }
+ } );
+ }
+ } );
+ }
+ });
}
-
@Override
public int getVersion() {
return edgeMigrationStrategy.getVersion();
}
- @Override
- public MigrationType getType() {
- return MigrationType.Applications;
- }
}
[48/50] [abbrv] incubator-usergrid git commit: tests passing in
queryindex
Posted by sf...@apache.org.
tests passing in queryindex
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/532b6620
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/532b6620
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/532b6620
Branch: refs/heads/USERGRID-480
Commit: 532b6620cd9769f39d5ba1cb119f66819fab5df1
Parents: 5f75292
Author: Shawn Feldman <sf...@apache.org>
Authored: Thu Mar 19 14:08:47 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Thu Mar 19 14:08:47 2015 -0600
----------------------------------------------------------------------
.../test/java/org/apache/usergrid/CoreApplication.java | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/532b6620/stack/core/src/test/java/org/apache/usergrid/CoreApplication.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/CoreApplication.java b/stack/core/src/test/java/org/apache/usergrid/CoreApplication.java
index a48ce4e..99c4654 100644
--- a/stack/core/src/test/java/org/apache/usergrid/CoreApplication.java
+++ b/stack/core/src/test/java/org/apache/usergrid/CoreApplication.java
@@ -145,11 +145,11 @@ public class CoreApplication implements Application, TestRule {
protected void after( Description description ) {
LOG.info( "Test {}: finish with application", description.getDisplayName() );
- try {
- setup.getEmf().getEntityManager(id).deleteIndex().get();
- }catch (Exception ee){
- throw new RuntimeException(ee);
- }
+// try {
+// setup.getEmf().getEntityManager(id).().get();
+// }catch (Exception ee){
+// throw new RuntimeException(ee);
+// }
}
[05/50] [abbrv] incubator-usergrid git commit: Temporary commit,
still refactoring modules and Guice configuration.
Posted by sf...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e96af464/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/MigrationInfoCache.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/MigrationInfoCache.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/MigrationInfoCache.java
new file mode 100644
index 0000000..480984f
--- /dev/null
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/MigrationInfoCache.java
@@ -0,0 +1,52 @@
+/*
+ *
+ * *
+ * * Licensed to the Apache Software Foundation (ASF) under one
+ * * or more contributor license agreements. See the NOTICE file
+ * * distributed with this work for additional information
+ * * regarding copyright ownership. The ASF licenses this file
+ * * to you under the Apache License, Version 2.0 (the
+ * * "License"); you may not use this file except in compliance
+ * * with the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing,
+ * * software distributed under the License is distributed on an
+ * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * * KIND, either express or implied. See the License for the
+ * * specific language governing permissions and limitations
+ * * under the License.
+ * *
+ *
+ */
+package org.apache.usergrid.persistence.core.migration.data;
+
+
+import org.apache.usergrid.persistence.core.migration.schema.Migration;
+
+
+/**
+ * A proxy that will cache migration info
+ */
+public interface MigrationInfoCache {
+
+ /**
+ * Invalidate the versions in theCache
+ */
+ public void invalidateAll();
+
+ /**
+ * Save the version for the plugin
+ * @param version
+ */
+ public void setVersion( final String pluginName, final int version );
+
+ /**
+ * Return the version
+ * @return
+ */
+ public int getVersion( final String pluginName );
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e96af464/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/MigrationInfoCacheImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/MigrationInfoCacheImpl.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/MigrationInfoCacheImpl.java
new file mode 100644
index 0000000..40dddaa
--- /dev/null
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/MigrationInfoCacheImpl.java
@@ -0,0 +1,84 @@
+/*
+ *
+ * *
+ * * Licensed to the Apache Software Foundation (ASF) under one
+ * * or more contributor license agreements. See the NOTICE file
+ * * distributed with this work for additional information
+ * * regarding copyright ownership. The ASF licenses this file
+ * * to you under the Apache License, Version 2.0 (the
+ * * "License"); you may not use this file except in compliance
+ * * with the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing,
+ * * software distributed under the License is distributed on an
+ * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * * KIND, either express or implied. See the License for the
+ * * specific language governing permissions and limitations
+ * * under the License.
+ * *
+ *
+ */
+
+package org.apache.usergrid.persistence.core.migration.data;
+
+
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+
+
+@Singleton
+public class MigrationInfoCacheImpl implements MigrationInfoCache{
+
+ /**
+ * Cache to cache versions temporarily
+ */
+ private final LoadingCache<String, Integer> versionCache = CacheBuilder.newBuilder()
+ //cache the local value for 1 minute
+ .expireAfterWrite( 1, TimeUnit.MINUTES ).build( new CacheLoader<String, Integer>() {
+ @Override
+ public Integer load( final String key ) throws Exception {
+ return migrationInfoSerialization.getVersion( key );
+ }
+ } );
+
+
+ private final MigrationInfoSerialization migrationInfoSerialization;
+
+
+ @Inject
+ public MigrationInfoCacheImpl( final MigrationInfoSerialization migrationInfoSerialization ) {
+ this.migrationInfoSerialization = migrationInfoSerialization;
+ }
+
+
+ @Override
+ public void invalidateAll() {
+ versionCache.invalidateAll();
+ }
+
+
+ @Override
+ public void setVersion( final String pluginName, final int version ) {
+ migrationInfoSerialization.setVersion( pluginName, version );
+ versionCache.invalidate( pluginName );
+ }
+
+
+ @Override
+ public int getVersion( final String pluginName ) {
+ try {
+ return versionCache.get( pluginName );
+ }
+ catch ( ExecutionException e ) {
+ throw new RuntimeException("Unable to get cached version for plugin name " + pluginName);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e96af464/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/MigrationRelationship.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/MigrationRelationship.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/MigrationRelationship.java
index a9ad187..691d8c1 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/MigrationRelationship.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/MigrationRelationship.java
@@ -25,17 +25,76 @@ package org.apache.usergrid.persistence.core.migration.data.newimpls;
/**
- * Simple relationship that defines the current state of the source and destination data versions
+ * Simple relationship that defines the current state of the source and destination data versions. Note that
+ * ina current system, the from and then to will be the same instance
*/
public class MigrationRelationship<T extends VersionedData> {
//public so it's FAST. It's also immutable
+
+
public final T from;
public final T to;
+ private final int fromVersion;
+ private final int toVersion;
+
public MigrationRelationship( T from, T to ) {
this.from = from;
this.to = to;
+
+ fromVersion = from.getImplementationVersion();
+ toVersion = to.getImplementationVersion();
+ }
+
+
+ /**
+ * Returns true if we need to perform dual writes. IE. the from is not the same as the to
+ * @return
+ */
+ public boolean needsMigration(){
+ return fromVersion != toVersion;
+ }
+
+
+ /**
+ * Return true if this is the migration relationship we should use. The version matches the from
+ * and is <= the to
+ * @param currentVersion
+ * @return
+ */
+ public boolean correctRelationship(final int currentVersion){
+ return currentVersion == fromVersion && currentVersion <= toVersion;
+ }
+
+
+ @Override
+ public boolean equals( final Object o ) {
+ if ( this == o ) {
+ return true;
+ }
+ if ( !( o instanceof MigrationRelationship ) ) {
+ return false;
+ }
+
+ final MigrationRelationship that = ( MigrationRelationship ) o;
+
+ if ( !from.equals( that.from ) ) {
+ return false;
+ }
+ if ( !to.equals( that.to ) ) {
+ return false;
+ }
+
+ return true;
+ }
+
+
+ @Override
+ public int hashCode() {
+ int result = from.hashCode();
+ result = 31 * result + to.hashCode();
+ return result;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e96af464/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/VersionedMigrationSet.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/VersionedMigrationSet.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/VersionedMigrationSet.java
new file mode 100644
index 0000000..f40f5d0
--- /dev/null
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/VersionedMigrationSet.java
@@ -0,0 +1,153 @@
+/*
+ *
+ * *
+ * * Licensed to the Apache Software Foundation (ASF) under one
+ * * or more contributor license agreements. See the NOTICE file
+ * * distributed with this work for additional information
+ * * regarding copyright ownership. The ASF licenses this file
+ * * to you under the Apache License, Version 2.0 (the
+ * * "License"); you may not use this file except in compliance
+ * * with the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing,
+ * * software distributed under the License is distributed on an
+ * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * * KIND, either express or implied. See the License for the
+ * * specific language governing permissions and limitations
+ * * under the License.
+ * *
+ *
+ */
+
+package org.apache.usergrid.persistence.core.migration.data.newimpls;
+
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import com.google.common.base.Preconditions;
+
+
+/**
+ * A set that represents a set of tuples that are used for
+ * @param <T>
+ */
+public class VersionedMigrationSet<T extends VersionedData> {
+
+
+ /**
+ * Cache so that after our initial lookup, it O(1) since this will be used heavily
+ *
+ */
+ private Map<Integer, MigrationRelationship<T>> cacheVersion = new HashMap<>();
+
+ private List<MigrationRelationship<T>> orderedVersions = new ArrayList<>();
+
+
+ /**
+ * Construct this set from a group of tuples. Imagine the following versions
+ *
+ * v1,
+ * v2,
+ * v3,
+ * v4
+ *
+ * Migrations can jump from v1->v3, but not directly to v4 without an extraneous migration. This would have 2 relationships
+ *
+ * v1, v3
+ * v2, v3
+ * and
+ * v3, v4
+ *
+ *
+ * @param migrations
+ */
+ public VersionedMigrationSet( final MigrationRelationship<T>... migrations ){
+ Preconditions.checkNotNull(migrations, "versions must not be null");
+ Preconditions.checkArgument( migrations.length > 0, "You must specify at least 1 migrationrelationship" );
+
+ orderedVersions.addAll( Arrays.asList(migrations ) );
+
+ Collections.sort( orderedVersions, new VersionedDataComparator() );
+
+ }
+
+
+ /**
+ * Get the migration relationship based on our current version. This will return a range that includes the current
+ * system version as the source, and the highest version we can roll to in the to field
+ * @param currentVersion
+ * @return The MigrationRelationship. Note the from and the to could be the same version in a current system.
+ */
+ public MigrationRelationship<T> getMigrationRelationship( final int currentVersion ){
+
+ final MigrationRelationship<T> relationship = cacheVersion.get( currentVersion );
+
+ if(relationship != null){
+ return relationship;
+ }
+
+ //not there, find it. Not the most efficient, but it happens once per version, which rarely changes, so not a big deal
+
+
+ for(MigrationRelationship<T> current: orderedVersions){
+
+ //not our instance, the from is too high
+ //our from is this instance, so we support this tuple. Our future is >= as well, so we can perform this I/O
+ if ( current.correctRelationship( currentVersion )) {
+ cacheVersion.put( currentVersion, current );
+ return current;
+ }
+
+ }
+
+ //if we get here, something is wrong
+ throw new IllegalArgumentException( "Could not find a migration version for version " + currentVersion + " min found was " + orderedVersions.get( orderedVersions.size()-1 ) );
+
+
+ }
+
+
+ /**
+ * Given the current system version, return the maximum migration version we can move to
+ * @param currentVersion
+ * @return
+ */
+ public int getMaxVersion(final int currentVersion){
+ return getMigrationRelationship( currentVersion ).to.getImplementationVersion();
+ }
+
+
+
+ /**
+ * Orders from high to low
+ */
+ private final class VersionedDataComparator implements Comparator<MigrationRelationship<T>>
+ {
+
+ @Override
+ public int compare( final MigrationRelationship<T> o1, final MigrationRelationship<T> o2 ) {
+ //Put higher target version first, since that's what we want to match based on current state and source
+
+ //order by the source. Put highest source first
+ int compare = Integer.compare( o1.to.getImplementationVersion(), o2.to.getImplementationVersion() ) *-1;
+
+
+ //put higher from first, if we fall within a range here we're good
+ if(compare == 0){
+ compare = Integer.compare( o1.from.getImplementationVersion(), o2.from.getImplementationVersion() ) *-1;
+ }
+
+ return compare;
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e96af464/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/VersionedSet.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/VersionedSet.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/VersionedSet.java
deleted file mode 100644
index a5123cf..0000000
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/VersionedSet.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- *
- * *
- * * Licensed to the Apache Software Foundation (ASF) under one
- * * or more contributor license agreements. See the NOTICE file
- * * distributed with this work for additional information
- * * regarding copyright ownership. The ASF licenses this file
- * * to you under the Apache License, Version 2.0 (the
- * * "License"); you may not use this file except in compliance
- * * with the License. You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing,
- * * software distributed under the License is distributed on an
- * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * * KIND, either express or implied. See the License for the
- * * specific language governing permissions and limitations
- * * under the License.
- * *
- *
- */
-
-package org.apache.usergrid.persistence.core.migration.data.newimpls;
-
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeMap;
-
-import com.google.inject.Inject;
-
-
-public class VersionedSet<T extends VersionedData> {
-
-
- /**
- * Cache so that after our initial lookup, it O(1) since this will be used heavily
- *
- */
- private Map<Integer, MigrationRelationship<T>> cacheVersion = new HashMap<>();
-
- private List<T> orderedVersions = new ArrayList<>();
-
-
-
-
- public VersionedSet(final Collection<T> versions){
-
- orderedVersions.addAll(versions );
- Collections.sort( orderedVersions, new VersionedDataComparator() );
-
- }
-
-
- /**
- * Get the migration relationship based on our current version
- * @param currentVersion
- * @return
- */
- public MigrationRelationship<T> getCurrentReadVersion(final int currentVersion){
-
- final MigrationRelationship<T> relationship = cacheVersion.get( currentVersion );
-
- if(relationship != null){
- return relationship;
- }
-
- //not there, find it. Not the most efficient, but it happens once per version, which rarely changes, so not a big deal
-
-
- for(T current: orderedVersions){
- //not our instance
- if(current.getImplementationVersion() > currentVersion){
- continue;
- }
-
-
- //we always go from our first match to our highest version. Any versions between can be skipped
- final MigrationRelationship<T> migrationRelationship = new MigrationRelationship<>( current, orderedVersions.get( 0 ) );
-
- cacheVersion.put( currentVersion, migrationRelationship );
-
- return migrationRelationship;
-
- }
-
- //if we get here, something is wrong
- throw new IllegalArgumentException( "Could not find a migration version for version " + currentVersion + " min found was " + orderedVersions.get( orderedVersions.size()-1 ) );
-
-
- }
-
-
- /**
- * Orders from high to low
- */
- private static final class VersionedDataComparator implements Comparator<VersionedData>
- {
-
- @Override
- public int compare( final VersionedData o1, final VersionedData o2 ) {
- return Integer.compare( o1.getImplementationVersion(), o2.getImplementationVersion())*-1;
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e96af464/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/rx/AllEntitiesInSystemObservable.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/rx/AllEntitiesInSystemObservable.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/rx/AllEntitiesInSystemObservable.java
deleted file mode 100644
index e7a7aca..0000000
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/rx/AllEntitiesInSystemObservable.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- *
- * * Licensed to the Apache Software Foundation (ASF) under one or more
- * * contributor license agreements. The ASF licenses this file to You
- * * under the Apache License, Version 2.0 (the "License"); you may not
- * * use this file except in compliance with the License.
- * * You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing, software
- * * distributed under the License is distributed on an "AS IS" BASIS,
- * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * * See the License for the specific language governing permissions and
- * * limitations under the License. For additional information regarding
- * * copyright in this work, please see the NOTICE file in the top level
- * * directory of this distribution.
- *
- */
-package org.apache.usergrid.persistence.core.rx;
-
-import org.apache.usergrid.persistence.core.scope.ApplicationEntityGroup;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-
-import rx.Observable;
-
-
-/**
- * An observable that will emit every entity Id stored in our entire system across all apps.
- * Note that this only walks each application applicationId graph, and emits edges from the applicationId and it's edges as the s
- * source node
- */
-public interface AllEntitiesInSystemObservable<T extends ApplicationScope> {
- /**
- * Return an observable that emits all entities in the system.
- *
- * @param bufferSize The amount of entityIds to buffer into each ApplicationEntityGroup. Note that if we exceed the buffer size
- * you may be more than 1 ApplicationEntityGroup with the same application and different ids
- */
- public Observable<ApplicationEntityGroup<T>> getAllEntitiesInSystem(final int bufferSize);
-
- /**
- * Return an observable that emits all entities in the system.
- *
- * @param appIdObservable list of app ids
- * @param bufferSize The amount of entityIds to buffer into each ApplicationEntityGroup. Note that if we exceed the buffer size
- * you may be more than 1 ApplicationEntityGroup with the same application and different ids
- */
- public Observable<ApplicationEntityGroup<T>> getAllEntitiesInSystem(Observable<ApplicationScope> appIdObservable, final int bufferSize);
-
-
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e96af464/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/rx/ApplicationObservable.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/rx/ApplicationObservable.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/rx/ApplicationObservable.java
deleted file mode 100644
index a229eb6..0000000
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/rx/ApplicationObservable.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- *
- * * Licensed to the Apache Software Foundation (ASF) under one or more
- * * contributor license agreements. The ASF licenses this file to You
- * * under the Apache License, Version 2.0 (the "License"); you may not
- * * use this file except in compliance with the License.
- * * You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing, software
- * * distributed under the License is distributed on an "AS IS" BASIS,
- * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * * See the License for the specific language governing permissions and
- * * limitations under the License. For additional information regarding
- * * copyright in this work, please see the NOTICE file in the top level
- * * directory of this distribution.
- *
- */
-package org.apache.usergrid.persistence.core.rx;
-
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.model.entity.Id;
-import rx.Observable;
-
-/**
- * Observable to emit all app ids
- */
-public interface ApplicationObservable {
- /**
- * Get all applicationIds as an observable
- */
- Observable<Id> getAllApplicationIds();
-
- /**
- * get all application scopes
- */
- Observable<ApplicationScope> getAllApplicationScopes();
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e96af464/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/guice/MaxMigrationModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/guice/MaxMigrationModule.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/guice/MaxMigrationModule.java
deleted file mode 100644
index 69b83b0..0000000
--- a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/guice/MaxMigrationModule.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.persistence.core.guice;
-
-
-import org.apache.usergrid.persistence.core.migration.data.ApplicationDataMigration;
-import org.apache.usergrid.persistence.core.migration.data.DataMigration;
-
-import com.google.inject.AbstractModule;
-import com.google.inject.multibindings.Multibinder;
-
-
-/**
- * Install this module in your tests if you want the max version to always be set
- * this ensures that the system will be on Integer.MAX version, there for
- */
-public class MaxMigrationModule extends AbstractModule {
- @Override
- protected void configure() {
- Multibinder<DataMigration> dataMigrationMultibinder = Multibinder.newSetBinder( binder(), DataMigration.class );
- dataMigrationMultibinder.addBinding().to( MaxMigrationVersion.class );
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e96af464/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/guice/MaxMigrationVersion.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/guice/MaxMigrationVersion.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/guice/MaxMigrationVersion.java
deleted file mode 100644
index 58cf5a6..0000000
--- a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/guice/MaxMigrationVersion.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.persistence.core.guice;
-
-
-import org.apache.usergrid.persistence.core.migration.data.ApplicationDataMigration;
-import org.apache.usergrid.persistence.core.scope.ApplicationEntityGroup;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import rx.Observable;
-
-
-/**
- * A simple migration that sets the version to max. This way our integration tests always test the latest code
- */
-public class MaxMigrationVersion implements ApplicationDataMigration {
-
- @Override
- public Observable migrate(final Observable<ApplicationScope> applicationEntityGroup, final ProgressObserver observer) {
- //no op, just needs to run to be set
- return Observable.empty();
- }
-
- @Override
- public int getVersion() {
- return Integer.MAX_VALUE;
- }
-
- @Override
- public MigrationType getType() {
- return MigrationType.Applications;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e96af464/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/guice/TestCommonModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/guice/TestCommonModule.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/guice/TestCommonModule.java
index 6972fc7..8585553 100644
--- a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/guice/TestCommonModule.java
+++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/guice/TestCommonModule.java
@@ -22,11 +22,6 @@
package org.apache.usergrid.persistence.core.guice;
-import org.apache.usergrid.persistence.core.rx.AllEntitiesInSystemObservable;
-import org.apache.usergrid.persistence.core.rx.AllEntitiesInSystemTestObservable;
-import org.apache.usergrid.persistence.core.rx.ApplicationObservable;
-import org.apache.usergrid.persistence.core.rx.ApplicationsTestObservable;
-
/**
* Module for testing common frameworks
*/
@@ -34,9 +29,6 @@ public class TestCommonModule extends TestModule {
@Override
protected void configure() {
- bind(ApplicationObservable.class).to(ApplicationsTestObservable.class);
-
- bind(AllEntitiesInSystemObservable.class).to(AllEntitiesInSystemTestObservable.class);
install(new CommonModule());
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e96af464/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/guice/TestModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/guice/TestModule.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/guice/TestModule.java
index 3aeb629..0e11c60 100644
--- a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/guice/TestModule.java
+++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/guice/TestModule.java
@@ -25,8 +25,6 @@ import java.io.IOException;
import com.google.inject.AbstractModule;
import com.netflix.config.ConfigurationManager;
-import org.apache.usergrid.persistence.core.scope.ApplicationEntityGroup;
-
public abstract class TestModule extends AbstractModule {
static {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e96af464/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManagerImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManagerImplTest.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManagerImplTest.java
index 20972f6..a6d5717 100644
--- a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManagerImplTest.java
+++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManagerImplTest.java
@@ -22,26 +22,9 @@
package org.apache.usergrid.persistence.core.migration.data;
-import java.util.*;
-
-import org.apache.usergrid.persistence.core.rx.AllEntitiesInSystemObservable;
-import org.apache.usergrid.persistence.core.rx.ApplicationObservable;
-import org.apache.usergrid.persistence.core.scope.ApplicationEntityGroup;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
-import org.apache.usergrid.persistence.core.scope.EntityIdScope;
-import org.apache.usergrid.persistence.model.entity.Id;
-import org.apache.usergrid.persistence.model.entity.SimpleId;
import org.junit.Test;
-import org.mockito.invocation.InvocationOnMock;
-import org.mockito.stubbing.Answer;
-
-import org.apache.usergrid.persistence.core.migration.schema.MigrationException;
-import rx.Observable;
-import rx.Subscriber;
-import static org.mockito.Matchers.any;
-import static org.mockito.Mockito.*;
+import static org.junit.Assert.fail;
/**
@@ -49,215 +32,217 @@ import static org.mockito.Mockito.*;
*/
public class DataMigrationManagerImplTest {
- AllEntitiesInSystemObservable allEntitiesInSystemObservable = new AllEntitiesInSystemObservable() {
- @Override
- public Observable<ApplicationEntityGroup> getAllEntitiesInSystem(int bufferSize) {
-
- return Observable.create(new Observable.OnSubscribe<ApplicationEntityGroup>() {
- @Override
- public void call(Subscriber<? super ApplicationEntityGroup> subscriber) {
- ApplicationEntityGroup entityGroup = new ApplicationEntityGroup(mock(ApplicationScope.class),new ArrayList<EntityIdScope>());
- subscriber.onNext(entityGroup);
- subscriber.onCompleted();
- }
- });
- }
-
- @Override
- public Observable<ApplicationEntityGroup> getAllEntitiesInSystem(Observable appIdObservable, int bufferSize) {
- return this.getAllEntitiesInSystem(bufferSize) ;
- }
- };
-
- ApplicationObservable allApplicationsObservable = new ApplicationObservable() {
- @Override
- public Observable<Id> getAllApplicationIds() {
- return Observable.just( (Id)new SimpleId("application"));
- }
-
- @Override
- public Observable<ApplicationScope> getAllApplicationScopes() {
- return Observable.just( (ApplicationScope)new ApplicationScopeImpl((Id)new SimpleId("application")));
- }
- };
-
- @Test
- public void noMigrations() throws MigrationException {
- final MigrationInfoSerialization serialization = mock( MigrationInfoSerialization.class );
- when(serialization.getCurrentVersion()).thenReturn(1);
-
- Set<DataMigration> emptyMigration = new HashSet<>();
-
- DataMigrationManagerImpl migrationManager = new DataMigrationManagerImpl( serialization, emptyMigration, allEntitiesInSystemObservable,allApplicationsObservable );
-
- migrationManager.migrate();
-
- verify( serialization, never() ).setStatusMessage( any( String.class ) );
- verify( serialization, never() ).setStatusCode( any( Integer.class ) );
- verify( serialization, never() ).setVersion( any( Integer.class ) );
- }
-
-
- @Test
- public void multipleMigrations() throws Throwable {
- final MigrationInfoSerialization serialization = mock( MigrationInfoSerialization.class );
- when(serialization.getCurrentVersion()).thenReturn(1);
-
-
- final ApplicationDataMigration v1 = mock( ApplicationDataMigration.class );
- when( v1.getVersion() ).thenReturn( 2 );
- when( v1.migrate(any(Observable.class), any(DataMigration.ProgressObserver.class))).thenReturn(Observable.empty());
-
- final ApplicationDataMigration v2 = mock( ApplicationDataMigration.class );
- when( v2.getVersion() ).thenReturn( 3 );
- when(v2.migrate(any(Observable.class), any(DataMigration.ProgressObserver.class))).thenReturn(Observable.empty());
-
-
- Set<DataMigration> migrations = new HashSet<>();
- migrations.add( v1 );
- migrations.add( v2 );
-
-
-
- DataMigrationManagerImpl migrationManager = new DataMigrationManagerImpl( serialization, migrations,allEntitiesInSystemObservable,allApplicationsObservable );
-
- migrationManager.migrate();
-
-
- verify( v1 ).migrate(any(Observable.class), any( DataMigration.ProgressObserver.class ) );
- verify( v2 ).migrate(any(Observable.class), any( DataMigration.ProgressObserver.class ) );
-
- //verify we set the running status
- verify( serialization, times( 2 ) ).setStatusCode( DataMigrationManagerImpl.StatusCode.RUNNING.status );
-
- //set the status message
- verify( serialization, times( 2 * 2 ) ).setStatusMessage( any( String.class ) );
-
- verify( serialization ).setStatusCode( DataMigrationManagerImpl.StatusCode.COMPLETE.status );
-
- //verify we set version 1
- verify( serialization ).setVersion( 2 );
-
- //verify we set version 2
- verify( serialization ).setVersion( 3 );
- }
-
-
- @Test
- public void shortCircuitVersionFails() throws Throwable {
- final MigrationInfoSerialization serialization = mock( MigrationInfoSerialization.class );
- when(serialization.getCurrentVersion()).thenReturn(1);
-
-
- final ApplicationDataMigration v1 = mock( ApplicationDataMigration.class,"mock1" );
- when( v1.getVersion() ).thenReturn( 2 );
- when( v1.getType() ).thenReturn(DataMigration.MigrationType.Entities);
- when( v1.migrate(any(Observable.class), any(DataMigration.ProgressObserver.class))).thenReturn(Observable.empty());
-
- //throw an exception
- when( v1.migrate(any(Observable.class),
- any(DataMigration.ProgressObserver.class) )).thenThrow(new RuntimeException( "Something bad happened" ));
-
- final ApplicationDataMigration v2 = mock( ApplicationDataMigration.class,"mock2" );
- when( v2.getType() ).thenReturn(DataMigration.MigrationType.Entities);
- when( v2.getVersion() ).thenReturn( 3 );
-
- Set<DataMigration> migrations = new HashSet<>();
- migrations.add( v1 );
- migrations.add( v2 );
-
- DataMigrationManagerImpl migrationManager
- = new DataMigrationManagerImpl( serialization, migrations,allEntitiesInSystemObservable,allApplicationsObservable );
-
- migrationManager.migrate();
-
-
- verify( v1 ).migrate( any(Observable.class),any( DataMigration.ProgressObserver.class ) );
-
- //verify we don't run migration
- verify( v2, never() ).migrate( any(Observable.class),any( DataMigration.ProgressObserver.class ) );
-
- //verify we set the running status
- verify( serialization, times( 1 ) ).setStatusCode( DataMigrationManagerImpl.StatusCode.RUNNING.status );
-
- //set the status message
- verify( serialization, times( 2 ) ).setStatusMessage( any( String.class ) );
-
- //verify we set an error
- verify( serialization ).setStatusCode( DataMigrationManagerImpl.StatusCode.ERROR.status );
-
- //verify we never set version 1
- verify( serialization, never() ).setVersion( 1 );
-
- //verify we never set version 2
- verify( serialization, never() ).setVersion( 2 );
- }
-
-
- @Test
- public void failStopsProgress() throws Throwable {
- final MigrationInfoSerialization serialization = mock(MigrationInfoSerialization.class);
- when(serialization.getCurrentVersion()).thenReturn(1);
-
- final CollectionDataMigration v1 = mock( CollectionDataMigration.class );
- when( v1.getVersion() ).thenReturn( 2 );
- when( v1.getType() ).thenReturn(DataMigration.MigrationType.Entities);
- when( v1.migrate(any(Observable.class), any(DataMigration.ProgressObserver.class))).thenReturn(Observable.empty());
-
- final int returnedCode = 100;
-
- final String reason = "test reason";
-
- //mark as fail but don't
- when(v1.migrate(any(Observable.class), any(DataMigration.ProgressObserver.class))).thenAnswer(
- new Answer<Object>() {
- @Override
- public Object answer(final InvocationOnMock invocation) throws Throwable {
- final DataMigration.ProgressObserver progressObserver =
- (DataMigration.ProgressObserver) invocation.getArguments()[1];
-
- progressObserver.failed(returnedCode, reason);
- return null;
- }
- }
-
- );
-
- final CollectionDataMigration v2 = mock( CollectionDataMigration.class );
- when( v2.getVersion() ).thenReturn( 3 );
- when( v2.getType() ).thenReturn(DataMigration.MigrationType.Entities);
- when(v2.migrate(any(Observable.class), any(DataMigration.ProgressObserver.class))).thenReturn(Observable.empty());
-
- Set<DataMigration> applicationDataMigrations = new HashSet<>();
- applicationDataMigrations.add( v1 );
- applicationDataMigrations.add(v2);
-
-
- DataMigrationManagerImpl migrationManager = new DataMigrationManagerImpl( serialization, applicationDataMigrations,allEntitiesInSystemObservable, allApplicationsObservable );
-
- migrationManager.migrate();
-
-
- verify( v1 ).migrate(any(Observable.class), any( DataMigration.ProgressObserver.class ) );
-
- //verify we don't run migration
- verify( v2, never() ).migrate( any(Observable.class),any( DataMigration.ProgressObserver.class ) );
-
- //verify we set the running status
- verify( serialization, times( 1 ) ).setStatusCode( DataMigrationManagerImpl.StatusCode.RUNNING.status );
-
- //set the status message
- verify( serialization ).setStatusMessage( "Migration version 2. Starting migration" );
-
- verify( serialization ).setStatusMessage( "Migration version 100. Failed to migrate, reason is appended. Error 'test reason'" );
-
- //verify we set an error
- verify( serialization, times(2) ).setStatusCode( DataMigrationManagerImpl.StatusCode.ERROR.status );
-
- //verify we never set version 1
- verify( serialization, never() ).setVersion( 1 );
-
- //verify we never set version 2
- verify( serialization, never() ).setVersion( 2 );
- }
+ //TODO USERGRID-405 fix this
+//
+// AllEntitiesInSystemObservable allEntitiesInSystemObservable = new AllEntitiesInSystemObservable() {
+// @Override
+// public Observable<ApplicationEntityGroup> getAllEntitiesInSystem(int bufferSize) {
+//
+// return Observable.create(new Observable.OnSubscribe<ApplicationEntityGroup>() {
+// @Override
+// public void call(Subscriber<? super ApplicationEntityGroup> subscriber) {
+// ApplicationEntityGroup entityGroup = new ApplicationEntityGroup(mock(ApplicationScope.class),new ArrayList<EntityIdScope>());
+// subscriber.onNext(entityGroup);
+// subscriber.onCompleted();
+// }
+// });
+// }
+//
+// @Override
+// public Observable<ApplicationEntityGroup> getAllEntitiesInSystem(Observable appIdObservable, int bufferSize) {
+// return this.getAllEntitiesInSystem(bufferSize) ;
+// }
+// };
+//
+// ApplicationObservable allApplicationsObservable = new ApplicationObservable() {
+// @Override
+// public Observable<Id> getAllApplicationIds() {
+// return Observable.just( (Id)new SimpleId("application"));
+// }
+//
+// @Override
+// public Observable<ApplicationScope> getAllApplicationScopes() {
+// return Observable.just( (ApplicationScope)new ApplicationScopeImpl((Id)new SimpleId("application")));
+// }
+// };
+//
+// @Test
+// public void noMigrations() throws MigrationException {
+// final MigrationInfoSerialization serialization = mock( MigrationInfoSerialization.class );
+// when(serialization.getCurrentVersion()).thenReturn(1);
+//
+// Set<DataMigration> emptyMigration = new HashSet<>();
+//
+// DataMigrationManagerImpl migrationManager = new DataMigrationManagerImpl( serialization, emptyMigration, allEntitiesInSystemObservable,allApplicationsObservable );
+//
+// migrationManager.migrate();
+//
+// verify( serialization, never() ).setStatusMessage( any( String.class ) );
+// verify( serialization, never() ).setStatusCode( any( Integer.class ) );
+// verify( serialization, never() ).setVersion( any( Integer.class ) );
+// }
+//
+//
+// @Test
+// public void multipleMigrations() throws Throwable {
+// final MigrationInfoSerialization serialization = mock( MigrationInfoSerialization.class );
+// when(serialization.getCurrentVersion()).thenReturn(1);
+//
+//
+// final ApplicationDataMigration v1 = mock( ApplicationDataMigration.class );
+// when( v1.getVersion() ).thenReturn( 2 );
+// when( v1.migrate(any(Observable.class), any(DataMigration.ProgressObserver.class))).thenReturn(Observable.empty());
+//
+// final ApplicationDataMigration v2 = mock( ApplicationDataMigration.class );
+// when( v2.getVersion() ).thenReturn( 3 );
+// when(v2.migrate(any(Observable.class), any(DataMigration.ProgressObserver.class))).thenReturn(Observable.empty());
+//
+//
+// Set<DataMigration> migrations = new HashSet<>();
+// migrations.add( v1 );
+// migrations.add( v2 );
+//
+//
+//
+// DataMigrationManagerImpl migrationManager = new DataMigrationManagerImpl( serialization, migrations,allEntitiesInSystemObservable,allApplicationsObservable );
+//
+// migrationManager.migrate();
+//
+//
+// verify( v1 ).migrate(any(Observable.class), any( DataMigration.ProgressObserver.class ) );
+// verify( v2 ).migrate(any(Observable.class), any( DataMigration.ProgressObserver.class ) );
+//
+// //verify we set the running status
+// verify( serialization, times( 2 ) ).setStatusCode( DataMigrationManagerImpl.StatusCode.RUNNING.status );
+//
+// //set the status message
+// verify( serialization, times( 2 * 2 ) ).setStatusMessage( any( String.class ) );
+//
+// verify( serialization ).setStatusCode( DataMigrationManagerImpl.StatusCode.COMPLETE.status );
+//
+// //verify we set version 1
+// verify( serialization ).setVersion( 2 );
+//
+// //verify we set version 2
+// verify( serialization ).setVersion( 3 );
+// }
+//
+//
+// @Test
+// public void shortCircuitVersionFails() throws Throwable {
+// final MigrationInfoSerialization serialization = mock( MigrationInfoSerialization.class );
+// when(serialization.getCurrentVersion()).thenReturn(1);
+//
+//
+// final ApplicationDataMigration v1 = mock( ApplicationDataMigration.class,"mock1" );
+// when( v1.getVersion() ).thenReturn( 2 );
+// when( v1.getType() ).thenReturn(DataMigration.MigrationType.Entities);
+// when( v1.migrate(any(Observable.class), any(DataMigration.ProgressObserver.class))).thenReturn(Observable.empty());
+//
+// //throw an exception
+// when( v1.migrate(any(Observable.class),
+// any(DataMigration.ProgressObserver.class) )).thenThrow(new RuntimeException( "Something bad happened" ));
+//
+// final ApplicationDataMigration v2 = mock( ApplicationDataMigration.class,"mock2" );
+// when( v2.getType() ).thenReturn(DataMigration.MigrationType.Entities);
+// when( v2.getVersion() ).thenReturn( 3 );
+//
+// Set<DataMigration> migrations = new HashSet<>();
+// migrations.add( v1 );
+// migrations.add( v2 );
+//
+// DataMigrationManagerImpl migrationManager
+// = new DataMigrationManagerImpl( serialization, migrations,allEntitiesInSystemObservable,allApplicationsObservable );
+//
+// migrationManager.migrate();
+//
+//
+// verify( v1 ).migrate( any(Observable.class),any( DataMigration.ProgressObserver.class ) );
+//
+// //verify we don't run migration
+// verify( v2, never() ).migrate( any(Observable.class),any( DataMigration.ProgressObserver.class ) );
+//
+// //verify we set the running status
+// verify( serialization, times( 1 ) ).setStatusCode( DataMigrationManagerImpl.StatusCode.RUNNING.status );
+//
+// //set the status message
+// verify( serialization, times( 2 ) ).setStatusMessage( any( String.class ) );
+//
+// //verify we set an error
+// verify( serialization ).setStatusCode( DataMigrationManagerImpl.StatusCode.ERROR.status );
+//
+// //verify we never set version 1
+// verify( serialization, never() ).setVersion( 1 );
+//
+// //verify we never set version 2
+// verify( serialization, never() ).setVersion( 2 );
+// }
+//
+//
+// @Test
+// public void failStopsProgress() throws Throwable {
+// final MigrationInfoSerialization serialization = mock(MigrationInfoSerialization.class);
+// when(serialization.getCurrentVersion()).thenReturn(1);
+//
+// final CollectionDataMigration v1 = mock( CollectionDataMigration.class );
+// when( v1.getVersion() ).thenReturn( 2 );
+// when( v1.getType() ).thenReturn(DataMigration.MigrationType.Entities);
+// when( v1.migrate(any(Observable.class), any(DataMigration.ProgressObserver.class))).thenReturn(Observable.empty());
+//
+// final int returnedCode = 100;
+//
+// final String reason = "test reason";
+//
+// //mark as fail but don't
+// when(v1.migrate(any(Observable.class), any(DataMigration.ProgressObserver.class))).thenAnswer(
+// new Answer<Object>() {
+// @Override
+// public Object answer(final InvocationOnMock invocation) throws Throwable {
+// final DataMigration.ProgressObserver progressObserver =
+// (DataMigration.ProgressObserver) invocation.getArguments()[1];
+//
+// progressObserver.failed(returnedCode, reason);
+// return null;
+// }
+// }
+//
+// );
+//
+// final CollectionDataMigration v2 = mock( CollectionDataMigration.class );
+// when( v2.getVersion() ).thenReturn( 3 );
+// when( v2.getType() ).thenReturn(DataMigration.MigrationType.Entities);
+// when(v2.migrate(any(Observable.class), any(DataMigration.ProgressObserver.class))).thenReturn(Observable.empty());
+//
+// Set<DataMigration> applicationDataMigrations = new HashSet<>();
+// applicationDataMigrations.add( v1 );
+// applicationDataMigrations.add(v2);
+//
+//
+// DataMigrationManagerImpl migrationManager = new DataMigrationManagerImpl( serialization, applicationDataMigrations,allEntitiesInSystemObservable, allApplicationsObservable );
+//
+// migrationManager.migrate();
+//
+//
+// verify( v1 ).migrate(any(Observable.class), any( DataMigration.ProgressObserver.class ) );
+//
+// //verify we don't run migration
+// verify( v2, never() ).migrate( any(Observable.class),any( DataMigration.ProgressObserver.class ) );
+//
+// //verify we set the running status
+// verify( serialization, times( 1 ) ).setStatusCode( DataMigrationManagerImpl.StatusCode.RUNNING.status );
+//
+// //set the status message
+// verify( serialization ).setStatusMessage( "Migration version 2. Starting migration" );
+//
+// verify( serialization ).setStatusMessage( "Migration version 100. Failed to migrate, reason is appended. Error 'test reason'" );
+//
+// //verify we set an error
+// verify( serialization, times(2) ).setStatusCode( DataMigrationManagerImpl.StatusCode.ERROR.status );
+//
+// //verify we never set version 1
+// verify( serialization, never() ).setVersion( 1 );
+//
+// //verify we never set version 2
+// verify( serialization, never() ).setVersion( 2 );
+// }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e96af464/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/MigrationInfoSerializationTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/MigrationInfoSerializationTest.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/MigrationInfoSerializationTest.java
index 598d107..7ac1fb2 100644
--- a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/MigrationInfoSerializationTest.java
+++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/MigrationInfoSerializationTest.java
@@ -23,6 +23,7 @@ package org.apache.usergrid.persistence.core.migration.data;
+import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -41,11 +42,15 @@ import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
+import static org.junit.Assert.fail;
+
@RunWith( ITRunner.class )
@UseModules( TestCommonModule.class )
+@Ignore()
public class MigrationInfoSerializationTest {
+ //TODO USERGRID-405 fix this
@Inject
@Rule
@@ -63,54 +68,53 @@ public class MigrationInfoSerializationTest {
@Inject
protected MigrationInfoSerialization migrationInfoSerialization;
-
-
- @Test
- public void basicTest() throws ConnectionException, MigrationException {
-
- //drop the column family, then run setup
- keyspace.dropColumnFamily( MigrationInfoSerializationImpl.CF_MIGRATION_INFO.getName() );
-
- migrationManager.migrate();
-
- //test getting nothing works
- final String emptyStatus = migrationInfoSerialization.getStatusMessage();
-
- assertNull(emptyStatus);
-
- final int unsavedVersion = migrationInfoSerialization.getVersion();
-
- assertEquals(0, unsavedVersion);
-
- final int statusCode = migrationInfoSerialization.getStatusCode();
-
- assertEquals(0, statusCode);
-
- //now update them
-
- final String savedStatus = "I'm a test status";
-
- migrationInfoSerialization.setStatusMessage( savedStatus );
-
- final String returnedStatus = migrationInfoSerialization.getStatusMessage();
-
- assertEquals("Correct status returned", savedStatus, returnedStatus);
-
-
- final int savedVersion = 100;
-
- migrationInfoSerialization.setVersion( savedVersion );
-
- final int returnedVersion = migrationInfoSerialization.getVersion();
-
- assertEquals("Correct version returned", savedVersion, returnedVersion);
-
- final int savedStatusCode = 200;
-
- migrationInfoSerialization.setStatusCode( savedStatusCode );
-
- final int returnedStatusCode = migrationInfoSerialization.getStatusCode();
-
- assertEquals("Status code was set correctly", savedStatusCode, returnedStatusCode);
- }
+//
+// @Test
+// public void basicTest() throws ConnectionException, MigrationException {
+//
+// //drop the column family, then run setup
+// keyspace.dropColumnFamily( MigrationInfoSerializationImpl.CF_MIGRATION_INFO.getName() );
+//
+// migrationManager.migrate();
+//
+// //test getting nothing works
+// final String emptyStatus = migrationInfoSerialization.getStatusMessage();
+//
+// assertNull(emptyStatus);
+//
+// final int unsavedVersion = migrationInfoSerialization.getVersion();
+//
+// assertEquals(0, unsavedVersion);
+//
+// final int statusCode = migrationInfoSerialization.getStatusCode();
+//
+// assertEquals(0, statusCode);
+//
+// //now update them
+//
+// final String savedStatus = "I'm a test status";
+//
+// migrationInfoSerialization.setStatusMessage( savedStatus );
+//
+// final String returnedStatus = migrationInfoSerialization.getStatusMessage();
+//
+// assertEquals("Correct status returned", savedStatus, returnedStatus);
+//
+//
+// final int savedVersion = 100;
+//
+// migrationInfoSerialization.setVersion( savedVersion );
+//
+// final int returnedVersion = migrationInfoSerialization.getVersion();
+//
+// assertEquals("Correct version returned", savedVersion, returnedVersion);
+//
+// final int savedStatusCode = 200;
+//
+// migrationInfoSerialization.setStatusCode( savedStatusCode );
+//
+// final int returnedStatusCode = migrationInfoSerialization.getStatusCode();
+//
+// assertEquals("Status code was set correctly", savedStatusCode, returnedStatusCode);
+// }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e96af464/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/newimpls/VersionedMigrationSetTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/newimpls/VersionedMigrationSetTest.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/newimpls/VersionedMigrationSetTest.java
new file mode 100644
index 0000000..e35409a
--- /dev/null
+++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/newimpls/VersionedMigrationSetTest.java
@@ -0,0 +1,194 @@
+/*
+ *
+ * *
+ * * Licensed to the Apache Software Foundation (ASF) under one
+ * * or more contributor license agreements. See the NOTICE file
+ * * distributed with this work for additional information
+ * * regarding copyright ownership. The ASF licenses this file
+ * * to you under the Apache License, Version 2.0 (the
+ * * "License"); you may not use this file except in compliance
+ * * with the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing,
+ * * software distributed under the License is distributed on an
+ * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * * KIND, either express or implied. See the License for the
+ * * specific language governing permissions and limitations
+ * * under the License.
+ * *
+ *
+ */
+
+package org.apache.usergrid.persistence.core.migration.data.newimpls;
+
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+
+public class VersionedMigrationSetTest {
+
+
+ /**
+ * a single version that goes from 1 to 3, and 2 to 3. then current as 3, 3. 1, 2 and 3, should have this in
+ * their range
+ */
+ @Test
+ public void singleVersion() {
+ //
+ final MigrationRelationship<TestVersionImpl> relationship1_3 =
+ new MigrationRelationship<>( new TestVersionImpl( 1 ), new TestVersionImpl( 3 ) );
+
+ final MigrationRelationship<TestVersionImpl> relationship2_3 =
+ new MigrationRelationship<>( new TestVersionImpl( 2 ), new TestVersionImpl( 3 ) );
+
+ //our current state, a necessity based on the data structure
+
+ final MigrationRelationship<TestVersionImpl> current =
+ new MigrationRelationship<>( new TestVersionImpl( 3 ), new TestVersionImpl( 3 ) );
+
+
+ final VersionedMigrationSet<TestVersionImpl> set =
+ new VersionedMigrationSet<>( relationship1_3, relationship2_3, current );
+
+
+ try {
+ set.getMigrationRelationship( 0 );
+ fail( "I should throw an exception" );
+ }
+ catch ( IllegalArgumentException iae ) {
+ //swallow, it's outside the range
+ }
+
+ MigrationRelationship<TestVersionImpl> migrationRelationship = set.getMigrationRelationship( 1 );
+
+ assertEquals( relationship1_3, migrationRelationship );
+
+
+ migrationRelationship = set.getMigrationRelationship( 2 );
+
+ assertEquals( relationship2_3, migrationRelationship );
+
+ migrationRelationship = set.getMigrationRelationship( 3 );
+
+ assertEquals( current, migrationRelationship );
+
+ try {
+ set.getMigrationRelationship( 4 );
+ fail( "I should throw an exception" );
+ }
+ catch ( IllegalArgumentException iae ) {
+ //swallow, it's outside the range
+ }
+ }
+
+
+ /**
+ * a single version that goes from 1 to 3. versions that go from 2 to 3, then a barrier migration that must be run.
+ * This can happen during a massive data change. From there we can move on from 3 forward, so we go from 3:6, 4:6,
+ * 5:6, and 6:6, This should mean an older system say on v2, can jump from v2 to v3, then from v3 directly to v6.
+ */
+ @Test
+ public void versionBounds() {
+ //
+ final MigrationRelationship<TestVersionImpl> relationship1_3 =
+ new MigrationRelationship<>( new TestVersionImpl( 1 ), new TestVersionImpl( 3 ) );
+
+ final MigrationRelationship<TestVersionImpl> relationship2_3 =
+ new MigrationRelationship<>( new TestVersionImpl( 2 ), new TestVersionImpl( 3 ) );
+
+ //our current state, a necessity based on the data structure
+ final MigrationRelationship<TestVersionImpl> relationship3_6 =
+ new MigrationRelationship<>( new TestVersionImpl( 3 ), new TestVersionImpl( 6 ) );
+
+
+ final MigrationRelationship<TestVersionImpl> relationship4_6 =
+ new MigrationRelationship<>( new TestVersionImpl( 4 ), new TestVersionImpl( 6 ) );
+
+
+ final MigrationRelationship<TestVersionImpl> relationship5_6 =
+ new MigrationRelationship<>( new TestVersionImpl( 5 ), new TestVersionImpl( 6 ) );
+
+
+ final MigrationRelationship<TestVersionImpl> current =
+ new MigrationRelationship<>( new TestVersionImpl( 6 ), new TestVersionImpl( 6 ) );
+
+
+ final VersionedMigrationSet<TestVersionImpl> set =
+ new VersionedMigrationSet<>( relationship1_3, relationship2_3, relationship3_6, relationship4_6,
+ relationship5_6, current );
+
+
+ try {
+ set.getMigrationRelationship( 0 );
+ fail( "I should throw an exception" );
+ }
+ catch ( IllegalArgumentException iae ) {
+ //swallow, it's outside the range
+ }
+
+ MigrationRelationship<TestVersionImpl> migrationRelationship = set.getMigrationRelationship( 1 );
+
+ assertEquals( relationship1_3, migrationRelationship );
+
+ migrationRelationship = set.getMigrationRelationship( 2 );
+
+ assertEquals( relationship2_3, migrationRelationship );
+
+
+ //now go from v3, we should get 3 to 6
+
+ migrationRelationship = set.getMigrationRelationship( 3 );
+
+ assertEquals( relationship3_6, migrationRelationship );
+
+ migrationRelationship = set.getMigrationRelationship( 4 );
+
+ assertEquals( relationship4_6, migrationRelationship );
+
+ migrationRelationship = set.getMigrationRelationship( 5 );
+
+ assertEquals( relationship5_6, migrationRelationship );
+
+ migrationRelationship = set.getMigrationRelationship( 6 );
+
+ assertEquals( current, migrationRelationship );
+
+
+ try {
+ set.getMigrationRelationship( 7 );
+ fail( "I should throw an exception" );
+ }
+ catch ( IllegalArgumentException iae ) {
+ //swallow, it's outside the range
+ }
+ }
+
+
+ @Test( expected = IllegalArgumentException.class )
+ public void testNoInput() {
+ new VersionedMigrationSet<TestVersionImpl>();
+ }
+
+
+ /**
+ * Create the test version impl. Just returns the version provided
+ */
+ private static final class TestVersionImpl implements VersionedData {
+
+ private final int version;
+
+
+ private TestVersionImpl( final int version ) {this.version = version;}
+
+
+ @Override
+ public int getImplementationVersion() {
+ return version;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e96af464/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/rx/AllEntitiesInSystemTestObservable.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/rx/AllEntitiesInSystemTestObservable.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/rx/AllEntitiesInSystemTestObservable.java
deleted file mode 100644
index bf244e0..0000000
--- a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/rx/AllEntitiesInSystemTestObservable.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- *
- * * Licensed to the Apache Software Foundation (ASF) under one or more
- * * contributor license agreements. The ASF licenses this file to You
- * * under the Apache License, Version 2.0 (the "License"); you may not
- * * use this file except in compliance with the License.
- * * You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing, software
- * * distributed under the License is distributed on an "AS IS" BASIS,
- * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * * See the License for the specific language governing permissions and
- * * limitations under the License. For additional information regarding
- * * copyright in this work, please see the NOTICE file in the top level
- * * directory of this distribution.
- *
- */
-package org.apache.usergrid.persistence.core.rx;
-
-import org.apache.usergrid.persistence.core.scope.ApplicationEntityGroup;
-import rx.Observable;
-
-/**
- * Test class to mock observable
- */
-public class AllEntitiesInSystemTestObservable implements AllEntitiesInSystemObservable{
-
- @Override
- public Observable<ApplicationEntityGroup> getAllEntitiesInSystem(int bufferSize) {
- return Observable.empty();
- }
-
- @Override
- public Observable<ApplicationEntityGroup> getAllEntitiesInSystem(Observable appIdObservable, int bufferSize) {
- return this.getAllEntitiesInSystem(bufferSize);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e96af464/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/rx/ApplicationsTestObservable.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/rx/ApplicationsTestObservable.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/rx/ApplicationsTestObservable.java
deleted file mode 100644
index b798c41..0000000
--- a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/rx/ApplicationsTestObservable.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- *
- * * Licensed to the Apache Software Foundation (ASF) under one or more
- * * contributor license agreements. The ASF licenses this file to You
- * * under the Apache License, Version 2.0 (the "License"); you may not
- * * use this file except in compliance with the License.
- * * You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing, software
- * * distributed under the License is distributed on an "AS IS" BASIS,
- * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * * See the License for the specific language governing permissions and
- * * limitations under the License. For additional information regarding
- * * copyright in this work, please see the NOTICE file in the top level
- * * directory of this distribution.
- *
- */
-package org.apache.usergrid.persistence.core.rx;
-
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.model.entity.Id;
-import rx.Observable;
-
-/**
- * Classy class class.
- */
-public class ApplicationsTestObservable implements ApplicationObservable {
- @Override
- public Observable<Id> getAllApplicationIds() {
- return Observable.empty();
- }
-
- @Override
- public Observable<ApplicationScope> getAllApplicationScopes() {
- return Observable.empty();
- }
-}
[13/50] [abbrv] incubator-usergrid git commit: Pushed migration tests
down to core tiers. WIP
Posted by sf...@apache.org.
Pushed migration tests down to core tiers. WIP
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/802dcdec
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/802dcdec
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/802dcdec
Branch: refs/heads/USERGRID-480
Commit: 802dcdec4405f4a358cef7d253fa5695d64769cc
Parents: 7925a4b
Author: Todd Nine <tn...@apigee.com>
Authored: Tue Mar 3 17:34:21 2015 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Mar 4 10:28:10 2015 -0700
----------------------------------------------------------------------
.../AllApplicationsObservable.java | 39 +++
.../usergrid/corepersistence/CoreModule.java | 80 ++++--
.../corepersistence/CpEntityManagerFactory.java | 60 +++--
.../events/EntityVersionDeletedHandler.java | 17 +-
.../migration/CoreDataVersions.java | 44 +++
.../migration/CoreMigrationPlugin.java | 59 ++++
.../migration/EntityTypeMappingMigration.java | 90 +++----
.../migration/MigrationModuleVersion.java | 112 ++++++++
.../corepersistence/migration/Versions.java | 24 --
.../rx/impl/AbstractGraphVisitorImpl.java | 110 ++++++++
.../rx/impl/AllApplicationsObservableImpl.java | 142 ++++++++++
.../rx/impl/AllEntitiesInSystemImpl.java | 62 +++++
.../impl/AllEntitiesInSystemObservableImpl.java | 127 ---------
.../rx/impl/AllNodesInGraphImpl.java | 55 ++++
.../rx/impl/ApplicationObservableImpl.java | 149 ----------
.../migration/EntityDataMigrationIT.java | 270 -------------------
.../migration/EntityTypeMappingMigrationIT.java | 1 -
.../migration/GraphShardVersionMigrationIT.java | 235 ----------------
.../migration/MigrationTestRule.java | 9 +-
.../migration/TestProgressObserver.java | 71 -----
.../rx/AllEntitiesInSystemObservableIT.java | 45 +---
.../rx/ApplicationObservableTestIT.java | 16 +-
.../persistence/collection/MvccLogEntry.java | 1 +
.../collection/guice/CollectionModule.java | 31 +--
.../persistence/collection/guice/Write.java | 17 --
.../collection/guice/WriteUpdate.java | 17 --
.../EntityCollectionManagerFactoryImpl.java | 9 +-
.../impl/EntityCollectionManagerImpl.java | 21 +-
.../collection/mvcc/stage/write/WriteStart.java | 6 +-
.../migration/MvccEntityDataMigrationImpl.java | 50 ----
.../mvcc/stage/write/WriteStartTest.java | 6 +-
.../MvccEntityDataMigrationV1ToV3ImplTest.java | 179 ++++++++++++
.../core/guice/DataMigrationResetRule.java | 88 ++++++
.../migration/data/TestProgressObserver.java | 71 +++++
.../persistence/core/util/IdGenerator.java | 51 ++++
.../persistence/graph/guice/GraphModule.java | 7 +-
.../impl/EdgeDataMigrationImpl.java | 152 -----------
.../impl/migration/EdgeDataMigrationImpl.java | 149 ++++++++++
.../impl/migration/GraphMigrationPlugin.java | 7 +-
.../serialization/impl/migration/GraphNode.java | 39 +++
.../persistence/graph/GraphManagerIT.java | 21 +-
.../persistence/graph/GraphManagerLoadTest.java | 13 +-
.../graph/GraphManagerShardConsistencyIT.java | 10 +-
.../graph/GraphManagerShardingIT.java | 13 +-
.../graph/GraphManagerStressTest.java | 11 +-
.../graph/guice/TestGraphModule.java | 5 +-
.../graph/impl/EdgeDeleteListenerTest.java | 11 +-
.../graph/impl/NodeDeleteListenerTest.java | 9 +-
.../graph/impl/stage/EdgeDeleteRepairTest.java | 7 +-
.../graph/impl/stage/EdgeMetaRepairTest.java | 29 +-
.../EdgeMetadataSerializationTest.java | 39 +--
.../EdgeSerializationChopTest.java | 7 +-
.../serialization/EdgeSerializationTest.java | 21 +-
.../serialization/NodeSerializationTest.java | 15 +-
.../migration/EdgeDataMigrationImplTest.java | 177 ++++++++++++
.../impl/shard/EdgeShardSerializationTest.java | 7 +-
.../impl/shard/NodeShardAllocationTest.java | 27 +-
.../impl/shard/NodeShardCacheTest.java | 7 +-
.../impl/shard/ShardGroupCompactionTest.java | 7 +-
.../shard/count/NodeShardApproximationTest.java | 9 +-
.../NodeShardCounterSerializationTest.java | 6 +-
.../shard/impl/ShardEntryGroupIteratorTest.java | 15 +-
...rceDirectedEdgeDescendingComparatorTest.java | 23 +-
...getDirectedEdgeDescendingComparatorTest.java | 23 +-
.../graph/test/util/EdgeTestUtils.java | 29 +-
65 files changed, 1767 insertions(+), 1492 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/802dcdec/stack/core/src/main/java/org/apache/usergrid/corepersistence/AllApplicationsObservable.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/AllApplicationsObservable.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/AllApplicationsObservable.java
new file mode 100644
index 0000000..1187cf2
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/AllApplicationsObservable.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence;
+
+
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.entities.Application;
+
+import rx.Observable;
+
+
+/**
+ * Interface for generating an observable of all ApplicationScope
+ */
+public interface AllApplicationsObservable {
+
+ /**
+ * Return all applications in our system
+ * @return
+ */
+ public Observable<ApplicationScope> getAllApplications();
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/802dcdec/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
index d7d7f2f..9f01feb 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
@@ -15,33 +15,38 @@
*/
package org.apache.usergrid.corepersistence;
-import com.google.inject.AbstractModule;
-import com.google.inject.Provider;
-import com.google.inject.multibindings.Multibinder;
-import org.apache.usergrid.corepersistence.migration.EntityTypeMappingMigration;
+import org.springframework.context.ApplicationContext;
+
import org.apache.usergrid.corepersistence.events.EntityDeletedHandler;
import org.apache.usergrid.corepersistence.events.EntityVersionCreatedHandler;
import org.apache.usergrid.corepersistence.events.EntityVersionDeletedHandler;
-import org.apache.usergrid.corepersistence.rx.impl.AllEntitiesInSystemObservableImpl;
-import org.apache.usergrid.corepersistence.rx.impl.ApplicationObservableImpl;
+import org.apache.usergrid.corepersistence.migration.CoreMigrationPlugin;
+import org.apache.usergrid.corepersistence.migration.EntityTypeMappingMigration;
+import org.apache.usergrid.corepersistence.rx.impl.AllEntitiesInSystemImpl;
+import org.apache.usergrid.corepersistence.rx.impl.AllNodesInGraphImpl;
+import org.apache.usergrid.corepersistence.rx.impl.AllApplicationsObservableImpl;
import org.apache.usergrid.persistence.EntityManagerFactory;
import org.apache.usergrid.persistence.collection.event.EntityDeleted;
import org.apache.usergrid.persistence.collection.event.EntityVersionCreated;
import org.apache.usergrid.persistence.collection.event.EntityVersionDeleted;
import org.apache.usergrid.persistence.collection.guice.CollectionModule;
+import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
+import org.apache.usergrid.persistence.collection.serialization.impl.migration.MvccEntityDataMigrationImpl;
import org.apache.usergrid.persistence.core.guice.CommonModule;
-import org.apache.usergrid.persistence.core.migration.data.CollectionDataMigration;
-import org.apache.usergrid.persistence.core.migration.data.DataMigration;
-import org.apache.usergrid.persistence.core.rx.AllEntitiesInSystemObservable;
-import org.apache.usergrid.persistence.core.rx.ApplicationObservable;
+import org.apache.usergrid.persistence.core.migration.data.newimpls.DataMigration2;
+import org.apache.usergrid.persistence.core.migration.data.newimpls.MigrationDataProvider;
+import org.apache.usergrid.persistence.core.migration.data.newimpls.MigrationPlugin;
import org.apache.usergrid.persistence.graph.guice.GraphModule;
+import org.apache.usergrid.persistence.graph.serialization.impl.migration.GraphNode;
import org.apache.usergrid.persistence.index.guice.IndexModule;
import org.apache.usergrid.persistence.map.guice.MapModule;
import org.apache.usergrid.persistence.queue.guice.QueueModule;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.context.ApplicationContext;
+
+import com.google.inject.AbstractModule;
+import com.google.inject.Provider;
+import com.google.inject.TypeLiteral;
+import com.google.inject.multibindings.Multibinder;
/**
@@ -70,19 +75,34 @@ public class CoreModule extends AbstractModule {
bind(EntityManagerFactory.class).toProvider( lazyEntityManagerFactoryProvider );
install( new CommonModule());
- install(new CollectionModule());
- install(new GraphModule());
+ install( new CollectionModule() {
+ /**
+ * configure our migration data provider for all entities in the system
+ */
+ @Override
+ public void configureMigrationProvider() {
+
+ bind(new TypeLiteral< MigrationDataProvider<EntityIdScope>>(){}).to(
+ AllEntitiesInSystemImpl.class );
+ }
+ } );
+ install( new GraphModule() {
+
+ /**
+ * Override the observable that needs to be used for migration
+ */
+ @Override
+ public void configureMigrationProvider() {
+ bind( new TypeLiteral<MigrationDataProvider<GraphNode>>() {} ).to(
+ AllNodesInGraphImpl.class );
+ }
+ } );
install(new IndexModule());
install(new MapModule());
install(new QueueModule());
bind(ManagerCache.class).to( CpManagerCache.class );
- bind(AllEntitiesInSystemObservable.class).to( AllEntitiesInSystemObservableImpl.class );
- bind(ApplicationObservable.class).to( ApplicationObservableImpl.class );
- Multibinder<DataMigration> dataMigrationMultibinder =
- Multibinder.newSetBinder( binder(), DataMigration.class );
- dataMigrationMultibinder.addBinding().to( EntityTypeMappingMigration.class );
Multibinder<EntityDeleted> entityBinder =
Multibinder.newSetBinder(binder(), EntityDeleted.class);
@@ -97,6 +117,26 @@ public class CoreModule extends AbstractModule {
versionCreatedMultibinder.addBinding().to(EntityVersionCreatedHandler.class);
+ /**
+ * Create our migrations for within our core plugin
+ *
+ */
+ Multibinder<DataMigration2<EntityIdScope>> dataMigrationMultibinder =
+ Multibinder.newSetBinder( binder(), new TypeLiteral<DataMigration2<EntityIdScope>>() {} );
+
+
+ dataMigrationMultibinder.addBinding().to( MvccEntityDataMigrationImpl.class );
+ dataMigrationMultibinder.addBinding().to( EntityTypeMappingMigration.class );
+
+
+ //wire up the collection migration plugin
+ Multibinder.newSetBinder( binder(), MigrationPlugin.class ).addBinding().to( CoreMigrationPlugin.class );
+
+ bind(AllApplicationsObservable.class).to(AllApplicationsObservableImpl.class);
+
+
+
+
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/802dcdec/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index 61f32c5..9d0b03a 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -15,12 +15,6 @@
*/
package org.apache.usergrid.corepersistence;
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheLoader;
-import com.google.common.cache.LoadingCache;
-import com.google.inject.Injector;
-import com.yammer.metrics.annotation.Metered;
-import static java.lang.String.CASE_INSENSITIVE_ORDER;
import java.util.Arrays;
import java.util.HashMap;
@@ -33,6 +27,12 @@ import java.util.TreeMap;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicBoolean;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.BeansException;
+import org.springframework.context.ApplicationContext;
+import org.springframework.context.ApplicationContextAware;
+
import org.apache.commons.lang.StringUtils;
import org.apache.usergrid.corepersistence.util.CpNamingUtils;
@@ -43,16 +43,15 @@ import org.apache.usergrid.persistence.EntityManager;
import org.apache.usergrid.persistence.EntityManagerFactory;
import org.apache.usergrid.persistence.EntityRef;
import org.apache.usergrid.persistence.Results;
-import static org.apache.usergrid.persistence.Schema.PROPERTY_NAME;
-import static org.apache.usergrid.persistence.Schema.TYPE_APPLICATION;
import org.apache.usergrid.persistence.cassandra.CassandraService;
import org.apache.usergrid.persistence.cassandra.CounterUtils;
import org.apache.usergrid.persistence.cassandra.Setup;
import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.collection.EntityCollectionManager;
import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl;
+import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
import org.apache.usergrid.persistence.core.migration.data.DataMigrationManager;
-import org.apache.usergrid.persistence.core.rx.AllEntitiesInSystemObservable;
+import org.apache.usergrid.persistence.core.migration.data.newimpls.MigrationDataProvider;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
import org.apache.usergrid.persistence.core.util.Health;
@@ -68,15 +67,24 @@ import org.apache.usergrid.persistence.index.EntityIndex;
import org.apache.usergrid.persistence.index.query.Query;
import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.entity.SimpleId;
-import org.apache.usergrid.utils.UUIDUtils;
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.beans.BeansException;
-import org.springframework.context.ApplicationContext;
-import org.springframework.context.ApplicationContextAware;
+import org.apache.usergrid.utils.UUIDUtils;
+
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import com.google.inject.Injector;
+import com.google.inject.Key;
+import com.google.inject.TypeLiteral;
+import com.yammer.metrics.annotation.Metered;
+
import rx.Observable;
+import static java.lang.String.CASE_INSENSITIVE_ORDER;
+
+import static org.apache.usergrid.persistence.Schema.PROPERTY_NAME;
+import static org.apache.usergrid.persistence.Schema.TYPE_APPLICATION;
+
/**
* Implement good-old Usergrid EntityManagerFactory with the new-fangled Core Persistence API.
@@ -107,7 +115,6 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
private ManagerCache managerCache;
- private AllEntitiesInSystemObservable allEntitiesInSystemObservable;
private DataMigrationManager dataMigrationManager;
@@ -171,10 +178,8 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
return managerCache;
}
- private AllEntitiesInSystemObservable getAllEntitiesObservable(){
- if(allEntitiesInSystemObservable==null)
- allEntitiesInSystemObservable = injector.getInstance(AllEntitiesInSystemObservable.class);
- return allEntitiesInSystemObservable;
+ private Observable<EntityIdScope> getAllEntitiesObservable(){
+ return injector.getInstance( Key.get(new TypeLiteral< MigrationDataProvider<EntityIdScope>>(){})).getData();
}
@@ -601,7 +606,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
public long performEntityCount() {
//TODO, this really needs to be a task that writes this data somewhere since this will get
//progressively slower as the system expands
- return (Long) getAllEntitiesObservable().getAllEntitiesInSystem(1000).longCount().toBlocking().last();
+ return (Long) getAllEntitiesObservable().longCount().toBlocking().last();
}
@@ -720,20 +725,25 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
@Override
public String getMigrateDataStatus() {
- return dataMigrationManager.getLastStatus();
+ //TODO USERGRID-405 return dataMigrationManager.getLastStatus();
+ return null;
}
@Override
public int getMigrateDataVersion() {
- return dataMigrationManager.getCurrentVersion();
+ //TODO USERGRID-405
+ //return dataMigrationManager.getCurrentVersion();
+ return 0;
}
@Override
public void setMigrationVersion( final int version ) {
- dataMigrationManager.resetToVersion( version );
- dataMigrationManager.invalidate();
+ //TODO USERGRID-405
+//
+// dataMigrationManager.resetToVersion( version );
+// dataMigrationManager.invalidate();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/802dcdec/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java
index 127ae46..65dacee 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java
@@ -24,6 +24,7 @@ import static org.apache.usergrid.corepersistence.CoreModule.EVENTS_DISABLED;
import org.apache.usergrid.persistence.EntityManagerFactory;
import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.collection.MvccEntity;
+import org.apache.usergrid.persistence.collection.MvccLogEntry;
import org.apache.usergrid.persistence.collection.event.EntityVersionDeleted;
import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
import org.apache.usergrid.persistence.index.EntityIndex;
@@ -53,9 +54,11 @@ public class EntityVersionDeletedHandler implements EntityVersionDeleted {
private EntityManagerFactory emf;
+
@Override
- public void versionDeleted(
- final CollectionScope scope, final Id entityId, final List<MvccEntity> entityVersions) {
+ public void versionDeleted( final CollectionScope scope, final Id entityId,
+ final List<MvccLogEntry> entityVersions ) {
+
// This check is for testing purposes and for a test that to be able to dynamically turn
// off and on delete previous versions so that it can test clean-up on read.
@@ -84,13 +87,12 @@ public class EntityVersionDeletedHandler implements EntityVersionDeleted {
scope.getName()
);
- rx.Observable.from(entityVersions)
- .subscribeOn(Schedulers.io())
+ rx.Observable.from( entityVersions )
.buffer(serializationFig.getBufferSize())
- .map(new Func1<List<MvccEntity>, List<MvccEntity>>() {
+ .map(new Func1<List<MvccLogEntry>, List<MvccLogEntry>>() {
@Override
- public List<MvccEntity> call(List<MvccEntity> entityList) {
- for (MvccEntity entity : entityList) {
+ public List<MvccLogEntry> call(List<MvccLogEntry> entityList) {
+ for (MvccLogEntry entity : entityList) {
eibatch.deindex(indexScope, entityId, entity.getVersion());
}
eibatch.execute();
@@ -99,4 +101,5 @@ public class EntityVersionDeletedHandler implements EntityVersionDeleted {
}).toBlocking().last();
}
+
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/802dcdec/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/CoreDataVersions.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/CoreDataVersions.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/CoreDataVersions.java
new file mode 100644
index 0000000..58ba6a3
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/CoreDataVersions.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.migration;
+
+
+/**
+ * Versions of data as they exist across our system
+ */
+public enum CoreDataVersions {
+ //even though this didn't really come first in time, we need to run this first in order to bring our system
+ //up to date so that our new migration module can proceed.
+
+ INITIAL(0),
+ MIGRATION_VERSION_FIX(1),
+ ID_MAP_FIX(2);
+
+
+ private final int version;
+
+
+ private CoreDataVersions( final int version ) {this.version = version;}
+
+
+ public int getVersion() {
+ return version;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/802dcdec/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/CoreMigrationPlugin.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/CoreMigrationPlugin.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/CoreMigrationPlugin.java
new file mode 100644
index 0000000..b95900f
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/CoreMigrationPlugin.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.migration;
+
+
+import java.util.Set;
+
+import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
+import org.apache.usergrid.persistence.core.migration.data.MigrationInfoSerialization;
+import org.apache.usergrid.persistence.core.migration.data.newimpls.AbstractMigrationPlugin;
+import org.apache.usergrid.persistence.core.migration.data.newimpls.DataMigration2;
+import org.apache.usergrid.persistence.core.migration.data.newimpls.MigrationDataProvider;
+
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+
+
+/**
+ * Migration plugin for the collection module
+ */
+@Singleton
+public class CoreMigrationPlugin extends AbstractMigrationPlugin<EntityIdScope> {
+
+ public static final String PLUGIN_NAME = "core-data";
+
+
+
+ @Inject
+ public CoreMigrationPlugin( final Set<DataMigration2<EntityIdScope>> entityDataMigrations,
+ final MigrationDataProvider<EntityIdScope> entityIdScopeDataMigrationProvider,
+ final MigrationInfoSerialization migrationInfoSerialization ) {
+ super( entityDataMigrations, entityIdScopeDataMigrationProvider, migrationInfoSerialization );
+ }
+
+
+ @Override
+ public String getName() {
+ return PLUGIN_NAME;
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/802dcdec/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/EntityTypeMappingMigration.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/EntityTypeMappingMigration.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/EntityTypeMappingMigration.java
index 4dbc373..80d7ebe 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/EntityTypeMappingMigration.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/EntityTypeMappingMigration.java
@@ -23,88 +23,88 @@ package org.apache.usergrid.corepersistence.migration;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicLong;
-
import org.apache.usergrid.corepersistence.ManagerCache;
import org.apache.usergrid.corepersistence.util.CpNamingUtils;
-import org.apache.usergrid.persistence.core.migration.data.CollectionDataMigration;
-import org.apache.usergrid.persistence.core.scope.ApplicationEntityGroup;
-import org.apache.usergrid.persistence.core.migration.data.DataMigration;
-import org.apache.usergrid.persistence.core.rx.AllEntitiesInSystemObservable;
-import org.apache.usergrid.persistence.core.scope.EntityIdScope;
+import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
+import org.apache.usergrid.persistence.core.migration.data.newimpls.DataMigration2;
+import org.apache.usergrid.persistence.core.migration.data.newimpls.MigrationDataProvider;
+import org.apache.usergrid.persistence.core.migration.data.newimpls.ProgressObserver;
import org.apache.usergrid.persistence.map.MapManager;
import org.apache.usergrid.persistence.map.MapScope;
-import org.apache.usergrid.persistence.model.entity.Id;
import com.google.inject.Inject;
import rx.Observable;
-import rx.Scheduler;
import rx.functions.Action1;
import rx.functions.Func1;
-import rx.schedulers.Schedulers;
/**
* Migration to ensure that our entity id is written into our map data
*/
-public class EntityTypeMappingMigration implements CollectionDataMigration {
+public class EntityTypeMappingMigration implements DataMigration2<EntityIdScope> {
private final ManagerCache managerCache;
- private final AllEntitiesInSystemObservable allEntitiesInSystemObservable;
+ private final MigrationDataProvider<EntityIdScope> allEntitiesInSystemObservable;
@Inject
- public EntityTypeMappingMigration( final ManagerCache managerCache, final AllEntitiesInSystemObservable allEntitiesInSystemObservable) {
- this.managerCache = managerCache;
+ public EntityTypeMappingMigration( final ManagerCache managerCache,
+ final MigrationDataProvider<EntityIdScope> allEntitiesInSystemObservable ) {
+ this.managerCache = managerCache;
this.allEntitiesInSystemObservable = allEntitiesInSystemObservable;
}
@Override
- public Observable migrate(final Observable<ApplicationEntityGroup> applicationEntityGroupObservable, final ProgressObserver observer) throws Throwable {
+ public int migrate( final int currentVersion, final MigrationDataProvider<EntityIdScope> migrationDataProvider,
+ final ProgressObserver observer ) {
final AtomicLong atomicLong = new AtomicLong();
- return applicationEntityGroupObservable.flatMap(new Func1<ApplicationEntityGroup, Observable<Long>>() {
- @Override
- public Observable call(final ApplicationEntityGroup applicationEntityGroup) {
- final MapScope ms = CpNamingUtils.getEntityTypeMapScope(applicationEntityGroup.applicationScope.getApplication());
-
- final MapManager mapManager = managerCache.getMapManager(ms);
- return Observable.from(applicationEntityGroup.entityIds)
- .subscribeOn(Schedulers.io())
- .map(new Func1<EntityIdScope, Long>() {
- @Override
- public Long call(EntityIdScope idScope) {
- final UUID entityUuid = idScope.getId().getUuid();
- final String entityType = idScope.getId().getType();
-
- mapManager.putString(entityUuid.toString(), entityType);
-
- if (atomicLong.incrementAndGet() % 100 == 0) {
- updateStatus(atomicLong, observer);
- }
- return atomicLong.get();
- }
- });
- }
- });
- }
+ return allEntitiesInSystemObservable.getData()
+ //process the entities in parallel
+ .parallel( new Func1<Observable<EntityIdScope>, Observable<EntityIdScope>>() {
+
+
+ @Override
+ public Observable<EntityIdScope> call( final Observable<EntityIdScope> entityIdScopeObservable ) {
- private void updateStatus( final AtomicLong counter, final ProgressObserver observer ) {
+ //for each entity observable, get the map scope and write it to the map
+ return entityIdScopeObservable.doOnNext( new Action1<EntityIdScope>() {
+ @Override
+ public void call( final EntityIdScope entityIdScope ) {
+ final MapScope ms = CpNamingUtils
+ .getEntityTypeMapScope( entityIdScope.getCollectionScope().getApplication() );
- observer.update( getVersion(), String.format( "Updated %d entities", counter.get() ) );
+ final MapManager mapManager = managerCache.getMapManager( ms );
+
+ final UUID entityUuid = entityIdScope.getId().getUuid();
+ final String entityType = entityIdScope.getId().getType();
+
+ mapManager.putString( entityUuid.toString(), entityType );
+
+ if ( atomicLong.incrementAndGet() % 100 == 0 ) {
+ observer.update( getMaxVersion(),
+ String.format( "Updated %d entities", atomicLong.get() ) );
+ }
+ }
+ } );
+ }
+ } ).count().toBlocking().last();
}
@Override
- public int getVersion() {
- return Versions.VERSION_1;
+ public boolean supports( final int currentVersion ) {
+ //we move from the migration version fix to the current version
+ return CoreDataVersions.MIGRATION_VERSION_FIX.getVersion() == currentVersion;
}
+
@Override
- public MigrationType getType() {
- return MigrationType.Entities;
+ public int getMaxVersion() {
+ return CoreDataVersions.ID_MAP_FIX.getVersion();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/802dcdec/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/MigrationModuleVersion.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/MigrationModuleVersion.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/MigrationModuleVersion.java
new file mode 100644
index 0000000..1c124ac
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/MigrationModuleVersion.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.migration;
+
+
+import org.apache.usergrid.persistence.collection.serialization.impl.MvccEntitySerializationStrategyV2Impl;
+import org.apache.usergrid.persistence.collection.serialization.impl.migration.CollectionMigrationPlugin;
+import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
+import org.apache.usergrid.persistence.core.migration.data.MigrationInfoSerialization;
+import org.apache.usergrid.persistence.core.migration.data.newimpls.DataMigration2;
+import org.apache.usergrid.persistence.core.migration.data.newimpls.MigrationDataProvider;
+import org.apache.usergrid.persistence.core.migration.data.newimpls.ProgressObserver;
+import org.apache.usergrid.persistence.graph.serialization.impl.EdgeMetadataSerializationV2Impl;
+import org.apache.usergrid.persistence.graph.serialization.impl.migration.GraphMigrationPlugin;
+
+import com.google.inject.Inject;
+
+
+/**
+ * Migration to set our module versions now that we've refactor for sub modules Keeps the EntityIdScope because it won't
+ * subscribe to the data provider.
+ */
+public class MigrationModuleVersion implements DataMigration2<EntityIdScope> {
+
+
+ /**
+ * The migration from 0 -> 1 that re-writes all the entity id's into the map module
+ */
+ private static final int ID_MIGRATION = 1;
+
+ /**
+ * The migration from 1-> 2 that shards our edge meta data
+ */
+ private static final int EDGE_SHARD_MIGRATION = 2;
+
+ /**
+ * The migration from 2-> 3 that fixed the short truncation bug
+ */
+ private static final int ENTITY_V2_MIGRATION = 3;
+
+ private final MigrationInfoSerialization migrationInfoSerialization;
+
+ private final MvccEntitySerializationStrategyV2Impl serializationStrategyV2;
+
+ private final EdgeMetadataSerializationV2Impl edgeMetadataSerializationV2;
+
+
+ @Inject
+ public MigrationModuleVersion( final MigrationInfoSerialization migrationInfoSerialization,
+ final MvccEntitySerializationStrategyV2Impl serializationStrategyV2,
+ final EdgeMetadataSerializationV2Impl edgeMetadataSerializationV2 ) {
+ this.migrationInfoSerialization = migrationInfoSerialization;
+ this.serializationStrategyV2 = serializationStrategyV2;
+ this.edgeMetadataSerializationV2 = edgeMetadataSerializationV2;
+ }
+
+
+ @Override
+ public int migrate( final int currentVersion, final MigrationDataProvider<EntityIdScope> migrationDataProvider,
+ final ProgressObserver observer ) {
+
+ //we ignore our current version, since it will always be 0
+ final int legacyVersion = migrationInfoSerialization.getSystemVersion();
+
+ //now we store versions for each of our modules
+
+ switch ( legacyVersion ) {
+ case ID_MIGRATION:
+ //no op, we need to migration everything in all modules
+ break;
+ //we need to set the version of the entity data, and our edge shard migration. The fall through (no break) is deliberate
+ case ENTITY_V2_MIGRATION:
+ migrationInfoSerialization.setVersion( CollectionMigrationPlugin.PLUGIN_NAME, serializationStrategyV2.getImplementationVersion() );
+ case EDGE_SHARD_MIGRATION:
+ //set our shard migration to the migrated version
+ migrationInfoSerialization.setVersion( GraphMigrationPlugin.PLUGIN_NAME, edgeMetadataSerializationV2.getImplementationVersion() );
+ break;
+ }
+
+ return CoreDataVersions.MIGRATION_VERSION_FIX.getVersion();
+ }
+
+
+ @Override
+ public boolean supports( final int currentVersion ) {
+ //we move from the migration version fix to the current version
+ return CoreDataVersions.INITIAL.getVersion() == currentVersion;
+ }
+
+
+ @Override
+ public int getMaxVersion() {
+ return CoreDataVersions.MIGRATION_VERSION_FIX.getVersion();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/802dcdec/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/Versions.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/Versions.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/Versions.java
index 7d05733..d921353 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/Versions.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/Versions.java
@@ -22,34 +22,10 @@
package org.apache.usergrid.corepersistence.migration;
-import org.apache.usergrid.persistence.core.guice.V2Impl;
-import org.apache.usergrid.persistence.core.guice.V3Impl;
-import org.apache.usergrid.persistence.graph.serialization.EdgeMigrationStrategy;
-
-
/**
* Simple class to hold the constants of all versions
*/
public class Versions {
- /**
- * Version 1 of our mappings
- */
- public static final int VERSION_1 = 1;
-
- /**
- * Version 2. Edge meta changes
- */
- public static final int VERSION_2 = EdgeMigrationStrategy.MIGRATION_VERSION;
-
- /**
- * Version 3. migrate from entity serialization 1 -> 2
- */
- public static final int VERSION_3 = V2Impl.MIGRATION_VERSION;
-
- /**
- * Version 4. migrate from entity serialization 1 -> 2
- */
- public static final int VERSION_4 = V3Impl.MIGRATION_VERSION;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/802dcdec/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AbstractGraphVisitorImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AbstractGraphVisitorImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AbstractGraphVisitorImpl.java
new file mode 100644
index 0000000..07b7a3d
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AbstractGraphVisitorImpl.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.rx.impl;
+
+
+import org.apache.usergrid.corepersistence.AllApplicationsObservable;
+import org.apache.usergrid.corepersistence.util.CpNamingUtils;
+import org.apache.usergrid.persistence.collection.CollectionScope;
+import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
+import org.apache.usergrid.persistence.core.migration.data.newimpls.MigrationDataProvider;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.graph.GraphManager;
+import org.apache.usergrid.persistence.graph.GraphManagerFactory;
+import org.apache.usergrid.persistence.graph.serialization.TargetIdObservable;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.google.inject.Inject;
+
+import rx.Observable;
+import rx.functions.Func1;
+
+
+/**
+ * An observable that will emit every entity Id stored in our entire system across all apps.
+ * Note that this only walks each application applicationId graph, and emits edges from the applicationId and it's edges as the s
+ * source node
+ */
+public abstract class AbstractGraphVisitorImpl<T> implements MigrationDataProvider<T> {
+
+ private final AllApplicationsObservable applicationObservable;
+ private final GraphManagerFactory graphManagerFactory;
+ private final TargetIdObservable targetIdObservable;
+
+ @Inject
+ public AbstractGraphVisitorImpl( AllApplicationsObservable applicationObservable,
+ GraphManagerFactory graphManagerFactory, TargetIdObservable targetIdObservable ) {
+
+ this.applicationObservable = applicationObservable;
+ this.graphManagerFactory = graphManagerFactory;
+ this.targetIdObservable = targetIdObservable;
+ }
+
+
+
+ @Override
+ public Observable<T> getData() {
+ return applicationObservable.getAllApplications().flatMap( new Func1<ApplicationScope, Observable<T>>() {
+ @Override
+ public Observable<T> call( final ApplicationScope applicationScope ) {
+ return getAllEntities( applicationScope );
+ }
+ } );
+
+ }
+
+
+ private Observable<T> getAllEntities(final ApplicationScope applicationScope) {
+ final GraphManager gm = graphManagerFactory.createEdgeManager(applicationScope);
+ final Id applicationId = applicationScope.getApplication();
+
+ //load all nodes that are targets of our application node. I.E.
+ // entities that have been saved
+ final Observable<Id> entityNodes =
+ targetIdObservable.getTargetNodes(gm, applicationId);
+
+ //emit Scope + ID
+
+ //create our application node to emit since it's an entity as well
+ final Observable<Id> applicationNode = Observable.just(applicationId);
+
+ //merge both the specified application node and the entity node
+ // so they all get used
+ return Observable
+ .merge( applicationNode, entityNodes ).
+ map( new Func1<Id, T>() {
+ @Override
+ public T call( final Id id ) {
+ return generateData(applicationScope, id);
+ }
+ } );
+ }
+
+
+ /**
+ * Generate the data for the observable stream from the scope and the node id
+ * @param applicationScope
+ * @param nodeId
+ * @return
+ */
+ protected abstract T generateData(final ApplicationScope applicationScope, final Id nodeId);
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/802dcdec/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllApplicationsObservableImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllApplicationsObservableImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllApplicationsObservableImpl.java
new file mode 100644
index 0000000..0fc5452
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllApplicationsObservableImpl.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.rx.impl;
+
+
+import java.util.Arrays;
+import java.util.UUID;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.usergrid.corepersistence.AllApplicationsObservable;
+import org.apache.usergrid.corepersistence.util.CpNamingUtils;
+import org.apache.usergrid.persistence.collection.CollectionScope;
+import org.apache.usergrid.persistence.collection.EntityCollectionManager;
+import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
+import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.graph.Edge;
+import org.apache.usergrid.persistence.graph.GraphManager;
+import org.apache.usergrid.persistence.graph.GraphManagerFactory;
+import org.apache.usergrid.persistence.graph.SearchByEdgeType;
+import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdgeType;
+import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.google.inject.Inject;
+
+import rx.Observable;
+import rx.functions.Func1;
+
+import static org.apache.usergrid.corepersistence.util.CpNamingUtils.getApplicationScope;
+
+
+/**
+ * An observable that will emit all application stored in the system.
+ */
+public class AllApplicationsObservableImpl implements AllApplicationsObservable {
+
+ private static final Logger logger = LoggerFactory.getLogger( AllApplicationsObservableImpl.class );
+ private final EntityCollectionManagerFactory entityCollectionManagerFactory;
+ private final GraphManagerFactory graphManagerFactory;
+
+ @Inject
+ public AllApplicationsObservableImpl( EntityCollectionManagerFactory entityCollectionManagerFactory,
+ GraphManagerFactory graphManagerFactory ){
+
+ this.entityCollectionManagerFactory = entityCollectionManagerFactory;
+ this.graphManagerFactory = graphManagerFactory;
+ }
+
+
+
+ @Override
+ public Observable<ApplicationScope> getAllApplications() {
+
+ //emit our 3 hard coded applications that are used the manage the system first.
+ //this way consumers can perform whatever work they need to on the root system first
+ final Observable<ApplicationScope> systemIds = Observable.from( Arrays
+ .asList( getApplicationScope( CpNamingUtils.DEFAULT_APPLICATION_ID ),
+ getApplicationScope( CpNamingUtils.MANAGEMENT_APPLICATION_ID ),
+ getApplicationScope( CpNamingUtils.SYSTEM_APP_ID ) ) );
+
+
+ final ApplicationScope appScope = getApplicationScope( CpNamingUtils.SYSTEM_APP_ID );
+
+ final CollectionScope appInfoCollectionScope =
+ new CollectionScopeImpl( appScope.getApplication(), appScope.getApplication(),
+ CpNamingUtils.getCollectionScopeNameFromCollectionName( CpNamingUtils.APPINFOS ) );
+
+ final EntityCollectionManager collectionManager =
+ entityCollectionManagerFactory.createCollectionManager( appInfoCollectionScope );
+
+
+ final GraphManager gm = graphManagerFactory.createEdgeManager(appScope);
+
+
+ String edgeType = CpNamingUtils.getEdgeTypeFromCollectionName( CpNamingUtils.APPINFOS );
+
+ Id rootAppId = appScope.getApplication();
+
+
+ //we have app infos. For each of these app infos, we have to load the application itself
+ Observable<ApplicationScope> appIds = gm.loadEdgesFromSource(
+ new SimpleSearchByEdgeType( rootAppId, edgeType, Long.MAX_VALUE, SearchByEdgeType.Order.DESCENDING,
+ null ) ).flatMap( new Func1<Edge, Observable<ApplicationScope>>() {
+ @Override
+ public Observable<ApplicationScope> call( final Edge edge ) {
+ //get the app info and load it
+ final Id appInfo = edge.getTargetNode();
+
+ return collectionManager.load( appInfo )
+ //filter out null entities
+ .filter( new Func1<Entity, Boolean>() {
+ @Override
+ public Boolean call( final Entity entity ) {
+ if ( entity == null ) {
+ logger.warn( "Encountered a null application info for id {}", appInfo );
+ return false;
+ }
+
+ return true;
+ }
+ } )
+ //get the id from the entity
+ .map( new Func1<Entity, ApplicationScope>() {
+
+
+ @Override
+ public ApplicationScope call( final Entity entity ) {
+
+ final UUID uuid = ( UUID ) entity.getField( "applicationUuid" ).getValue();
+
+ return getApplicationScope( uuid );
+ }
+ } );
+ }
+ } );
+
+ return Observable.merge( systemIds, appIds );
+ }
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/802dcdec/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntitiesInSystemImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntitiesInSystemImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntitiesInSystemImpl.java
new file mode 100644
index 0000000..758a8aa
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntitiesInSystemImpl.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.rx.impl;
+
+
+import org.apache.usergrid.corepersistence.AllApplicationsObservable;
+import org.apache.usergrid.corepersistence.util.CpNamingUtils;
+import org.apache.usergrid.persistence.collection.CollectionScope;
+import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.graph.GraphManagerFactory;
+import org.apache.usergrid.persistence.graph.serialization.TargetIdObservable;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+
+
+/**
+ * An observable that will emit every entity Id stored in our entire system across all apps.
+ * Note that this only walks each application applicationId graph, and emits edges from the applicationId and it's edges as the s
+ * source node
+ */
+@Singleton
+public class AllEntitiesInSystemImpl extends AbstractGraphVisitorImpl<EntityIdScope> {
+
+
+ @Inject
+ public AllEntitiesInSystemImpl( final AllApplicationsObservable applicationObservable,
+ final GraphManagerFactory graphManagerFactory,
+ final TargetIdObservable targetIdObservable ) {
+ super( applicationObservable, graphManagerFactory, targetIdObservable );
+ }
+
+
+ @Override
+ protected EntityIdScope generateData( final ApplicationScope applicationScope, final Id nodeId ) {
+ CollectionScope scope =
+ CpNamingUtils.getCollectionScopeNameFromEntityType( applicationScope.getApplication(), nodeId.getType() );
+
+ final EntityIdScope idScope = new EntityIdScope( scope, nodeId );
+
+ return idScope;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/802dcdec/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntitiesInSystemObservableImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntitiesInSystemObservableImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntitiesInSystemObservableImpl.java
deleted file mode 100644
index 55667cb..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntitiesInSystemObservableImpl.java
+++ /dev/null
@@ -1,127 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.rx.impl;
-
-
-import java.util.ArrayList;
-import java.util.List;
-
-import com.google.inject.Inject;
-import org.apache.usergrid.corepersistence.util.CpNamingUtils;
-import org.apache.usergrid.persistence.collection.CollectionScope;
-import org.apache.usergrid.persistence.core.rx.AllEntitiesInSystemObservable;
-import org.apache.usergrid.persistence.core.rx.ApplicationObservable;
-import org.apache.usergrid.persistence.core.scope.ApplicationEntityGroup;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
-import org.apache.usergrid.persistence.core.scope.EntityIdScope;
-import org.apache.usergrid.persistence.graph.GraphManager;
-import org.apache.usergrid.persistence.graph.GraphManagerFactory;
-import org.apache.usergrid.persistence.graph.serialization.TargetIdObservable;
-import org.apache.usergrid.persistence.model.entity.Id;
-
-import rx.Observable;
-import rx.functions.Func1;
-
-
-/**
- * An observable that will emit every entity Id stored in our entire system across all apps.
- * Note that this only walks each application applicationId graph, and emits edges from the applicationId and it's edges as the s
- * source node
- */
-public class AllEntitiesInSystemObservableImpl implements AllEntitiesInSystemObservable<CollectionScope> {
-
- private final ApplicationObservable applicationObservable;
- private final GraphManagerFactory graphManagerFactory;
- private final TargetIdObservable targetIdObservable;
-
- @Inject
- public AllEntitiesInSystemObservableImpl(ApplicationObservable applicationObservable, GraphManagerFactory graphManagerFactory, TargetIdObservable targetIdObservable) {
-
- this.applicationObservable = applicationObservable;
- this.graphManagerFactory = graphManagerFactory;
- this.targetIdObservable = targetIdObservable;
- }
-
- public Observable<ApplicationEntityGroup<CollectionScope>> getAllEntitiesInSystem(final int bufferSize) {
- return getAllEntitiesInSystem(applicationObservable.getAllApplicationIds().map(new Func1<Id, ApplicationScope>() {
- @Override
- public ApplicationScope call(Id id) {
- return new ApplicationScopeImpl(id);
- }
- }), bufferSize);
-
- }
-
- public Observable<ApplicationEntityGroup<CollectionScope>> getAllEntitiesInSystem(final Observable<ApplicationScope> appIdsObservable, final int bufferSize) {
- //traverse all nodes in the graph, load all source edges from them, then re-save the meta data
- return appIdsObservable
- .flatMap(new Func1<ApplicationScope, Observable<ApplicationEntityGroup<CollectionScope>>>() {
- @Override
- public Observable<ApplicationEntityGroup<CollectionScope>> call(final ApplicationScope applicationScope) {
- return getAllEntities(applicationScope, bufferSize);
- }
- });
- }
-
- private Observable<ApplicationEntityGroup<CollectionScope>> getAllEntities(final ApplicationScope applicationScope, final int bufferSize) {
- final GraphManager gm = graphManagerFactory.createEdgeManager(applicationScope);
- final Id applicationId = applicationScope.getApplication();
-
- //load all nodes that are targets of our application node. I.E.
- // entities that have been saved
- final Observable<Id> entityNodes =
- targetIdObservable.getTargetNodes(gm, applicationId);
-
-
- //get scope here
-
-
- //emit Scope + ID
-
- //create our application node to emit since it's an entity as well
- final Observable<Id> applicationNode = Observable.just(applicationId);
-
- //merge both the specified application node and the entity node
- // so they all get used
- return Observable
- .merge(applicationNode, entityNodes)
- .buffer(bufferSize)
- .map(new Func1<List<Id>, List<EntityIdScope<CollectionScope>>>() {
- @Override
- public List<EntityIdScope<CollectionScope>> call(List<Id> ids) {
- List<EntityIdScope<CollectionScope>> scopes = new ArrayList<>(ids.size());
- for (Id id : ids) {
- CollectionScope scope = CpNamingUtils.getCollectionScopeNameFromEntityType(applicationId, id.getType());
- EntityIdScope<CollectionScope> idScope = new EntityIdScope<>(id, scope);
- scopes.add(idScope);
- }
- return scopes;
- }
- })
- .map(new Func1<List<EntityIdScope<CollectionScope>>, ApplicationEntityGroup<CollectionScope>>() {
- @Override
- public ApplicationEntityGroup<CollectionScope> call(final List<EntityIdScope<CollectionScope>> scopes) {
- return new ApplicationEntityGroup<>(applicationScope, scopes);
- }
- });
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/802dcdec/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllNodesInGraphImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllNodesInGraphImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllNodesInGraphImpl.java
new file mode 100644
index 0000000..19292b8
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllNodesInGraphImpl.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.rx.impl;
+
+
+import org.apache.usergrid.corepersistence.AllApplicationsObservable;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.graph.GraphManagerFactory;
+import org.apache.usergrid.persistence.graph.serialization.TargetIdObservable;
+import org.apache.usergrid.persistence.graph.serialization.impl.migration.GraphNode;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+
+
+/**
+ * An observable that will emit every entity's node Id stored in our entire system across all apps.
+ * Note that this only walks each application applicationId graph, and emits edges from the applicationId and it's edges as the s
+ * source node
+ */
+@Singleton
+public class AllNodesInGraphImpl extends AbstractGraphVisitorImpl<GraphNode> {
+
+
+ @Inject
+ public AllNodesInGraphImpl( final AllApplicationsObservable applicationObservable,
+ final GraphManagerFactory graphManagerFactory,
+ final TargetIdObservable targetIdObservable ) {
+ super( applicationObservable, graphManagerFactory, targetIdObservable );
+ }
+
+
+ @Override
+ protected GraphNode generateData( final ApplicationScope applicationScope, final Id nodeId ) {
+ return new GraphNode( applicationScope, nodeId );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/802dcdec/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/ApplicationObservableImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/ApplicationObservableImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/ApplicationObservableImpl.java
deleted file mode 100644
index f4a1057..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/ApplicationObservableImpl.java
+++ /dev/null
@@ -1,149 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.rx.impl;
-
-
-import java.util.Arrays;
-import java.util.UUID;
-
-import com.google.inject.Inject;
-import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
-import org.apache.usergrid.persistence.core.rx.ApplicationObservable;
-import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
-import org.apache.usergrid.persistence.graph.GraphManagerFactory;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.usergrid.corepersistence.util.CpNamingUtils;
-import org.apache.usergrid.persistence.collection.CollectionScope;
-import org.apache.usergrid.persistence.collection.EntityCollectionManager;
-import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.graph.Edge;
-import org.apache.usergrid.persistence.graph.GraphManager;
-import org.apache.usergrid.persistence.graph.SearchByEdgeType;
-import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdgeType;
-import org.apache.usergrid.persistence.model.entity.Entity;
-import org.apache.usergrid.persistence.model.entity.Id;
-
-import rx.Observable;
-import rx.functions.Func1;
-
-import static org.apache.usergrid.corepersistence.util.CpNamingUtils.generateApplicationId;
-import static org.apache.usergrid.corepersistence.util.CpNamingUtils.getApplicationScope;
-
-
-/**
- * An observable that will emit all application stored in the system.
- */
-public class ApplicationObservableImpl implements ApplicationObservable {
-
- private static final Logger logger = LoggerFactory.getLogger( ApplicationObservableImpl.class );
- private final EntityCollectionManagerFactory entityCollectionManagerFactory;
- private final GraphManagerFactory graphManagerFactory;
-
- @Inject
- public ApplicationObservableImpl(EntityCollectionManagerFactory entityCollectionManagerFactory, GraphManagerFactory graphManagerFactory){
-
- this.entityCollectionManagerFactory = entityCollectionManagerFactory;
- this.graphManagerFactory = graphManagerFactory;
- }
-
-
- @Override
- public Observable<Id> getAllApplicationIds() {
-
- //emit our 3 hard coded applications that are used the manage the system first.
- //this way consumers can perform whatever work they need to on the root system first
- final Observable<Id> systemIds = Observable.from( Arrays
- .asList( generateApplicationId( CpNamingUtils.DEFAULT_APPLICATION_ID ),
- generateApplicationId( CpNamingUtils.MANAGEMENT_APPLICATION_ID ),
- generateApplicationId( CpNamingUtils.SYSTEM_APP_ID ) ) );
-
-
- final ApplicationScope appScope = getApplicationScope( CpNamingUtils.SYSTEM_APP_ID );
-
- final CollectionScope appInfoCollectionScope =
- new CollectionScopeImpl( appScope.getApplication(), appScope.getApplication(),
- CpNamingUtils.getCollectionScopeNameFromCollectionName( CpNamingUtils.APPINFOS ) );
-
- final EntityCollectionManager collectionManager =
- entityCollectionManagerFactory.createCollectionManager( appInfoCollectionScope );
-
-
- final GraphManager gm = graphManagerFactory.createEdgeManager(appScope);
-
-
- String edgeType = CpNamingUtils.getEdgeTypeFromCollectionName( CpNamingUtils.APPINFOS );
-
- Id rootAppId = appScope.getApplication();
-
-
- //we have app infos. For each of these app infos, we have to load the application itself
- Observable<Id> appIds = gm.loadEdgesFromSource(
- new SimpleSearchByEdgeType( rootAppId, edgeType, Long.MAX_VALUE, SearchByEdgeType.Order.DESCENDING,
- null ) ).flatMap( new Func1<Edge, Observable<Id>>() {
- @Override
- public Observable<Id> call( final Edge edge ) {
- //get the app info and load it
- final Id appInfo = edge.getTargetNode();
-
- return collectionManager.load( appInfo )
- //filter out null entities
- .filter( new Func1<Entity, Boolean>() {
- @Override
- public Boolean call( final Entity entity ) {
- if ( entity == null ) {
- logger.warn( "Encountered a null application info for id {}", appInfo );
- return false;
- }
-
- return true;
- }
- } )
- //get the id from the entity
- .map( new Func1<org.apache.usergrid.persistence.model.entity.Entity, Id>() {
-
-
- @Override
- public Id call( final org.apache.usergrid.persistence.model.entity.Entity entity ) {
-
- final UUID uuid = ( UUID ) entity.getField( "applicationUuid" ).getValue();
-
- return CpNamingUtils.generateApplicationId( uuid );
- }
- } );
- }
- } );
-
- return Observable.merge( systemIds, appIds );
- }
-
- @Override
- public Observable<ApplicationScope> getAllApplicationScopes() {
- return getAllApplicationIds().map(new Func1<Id, ApplicationScope>() {
- @Override
- public ApplicationScope call(Id id) {
- ApplicationScope scope = new ApplicationScopeImpl(id);
- return scope;
- }
- });
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/802dcdec/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/EntityDataMigrationIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/EntityDataMigrationIT.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/EntityDataMigrationIT.java
deleted file mode 100644
index 4d87f8b..0000000
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/EntityDataMigrationIT.java
+++ /dev/null
@@ -1,270 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.migration;
-
-
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.Set;
-
-import org.apache.usergrid.persistence.collection.mvcc.MvccEntityMigrationStrategy;
-import org.apache.usergrid.persistence.collection.serialization.impl.MvccEntityDataMigrationImpl;
-import org.apache.usergrid.persistence.collection.serialization.impl.MvccEntitySerializationStrategyProxyV2Impl;
-import org.apache.usergrid.persistence.core.migration.data.*;
-import org.apache.usergrid.persistence.core.rx.AllEntitiesInSystemObservable;
-import org.apache.usergrid.persistence.core.rx.ApplicationObservable;
-import org.apache.usergrid.persistence.core.scope.ApplicationEntityGroup;
-import org.apache.usergrid.persistence.core.scope.EntityIdScope;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Rule;
-import org.junit.Test;
-
-import org.apache.usergrid.AbstractCoreIT;
-import org.apache.usergrid.cassandra.SpringResource;
-import org.apache.usergrid.corepersistence.EntityWriteHelper;
-import org.apache.usergrid.corepersistence.util.CpNamingUtils;
-import org.apache.usergrid.persistence.Entity;
-import org.apache.usergrid.persistence.EntityManager;
-import org.apache.usergrid.persistence.EntityManagerFactory;
-import org.apache.usergrid.persistence.SimpleEntityRef;
-import org.apache.usergrid.persistence.collection.CollectionScope;
-import org.apache.usergrid.persistence.collection.MvccEntity;
-import org.apache.usergrid.persistence.collection.serialization.MvccEntitySerializationStrategy;
-import org.apache.usergrid.persistence.model.entity.Id;
-import org.apache.usergrid.persistence.model.util.UUIDGenerator;
-
-import com.google.inject.Injector;
-import com.google.inject.Key;
-
-import net.jcip.annotations.NotThreadSafe;
-
-import rx.functions.Action1;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
-
-@NotThreadSafe
-public class EntityDataMigrationIT extends AbstractCoreIT {
-
-
- private Injector injector;
-
-
- private CollectionDataMigration entityDataMigration;
- private DataMigrationManager dataMigrationManager;
- private MigrationInfoSerialization migrationInfoSerialization;
- private MvccEntitySerializationStrategy v1Strategy;
- private MvccEntitySerializationStrategy v2Strategy;
- private EntityManagerFactory emf;
-
-
-
- /**
- * Rule to do the resets we need
- */
- @Rule
- public MigrationTestRule migrationTestRule =
-
- new MigrationTestRule( app, SpringResource.getInstance().getBean( Injector.class ) ,MvccEntityDataMigrationImpl.class );
- private AllEntitiesInSystemObservable allEntitiesInSystemObservable;
- private ApplicationObservable applicationObservable;
-
-
- @Before
- public void setup() {
- emf = setup.getEmf();
- injector = SpringResource.getInstance().getBean(Injector.class);
- entityDataMigration = injector.getInstance( MvccEntityDataMigrationImpl.class );
- injector = SpringResource.getInstance().getBean( Injector.class );
- dataMigrationManager = injector.getInstance( DataMigrationManager.class );
- migrationInfoSerialization = injector.getInstance( MigrationInfoSerialization.class );
- MvccEntityMigrationStrategy strategy = injector.getInstance(Key.get(MvccEntityMigrationStrategy.class));
- allEntitiesInSystemObservable = injector.getInstance(AllEntitiesInSystemObservable.class);
- applicationObservable = injector.getInstance(ApplicationObservable.class);
-
- v1Strategy = strategy.getMigration().from();
- v2Strategy = strategy.getMigration().to();
- }
-
-
- @Test
- @Ignore("Awaiting fix for USERGRID-268")
- public void testDataMigration() throws Throwable {
-
- assertEquals( "version 3 expected", 3, entityDataMigration.getVersion() );
- assertEquals( "Previous version expected", 2, dataMigrationManager.getCurrentVersion());
-
-
- final EntityManager newAppEm = app.getEntityManager();
-
- final String type1 = "type1thing";
- final String type2 = "type2thing";
- final int size = 10;
-
- final Set<Id> type1Identities = EntityWriteHelper.createTypes( newAppEm, type1, size );
- final Set<Id> type2Identities = EntityWriteHelper.createTypes( newAppEm, type2, size );
-
-
- final Set<Id> createdEntityIds = new HashSet<>();
- createdEntityIds.addAll( type1Identities );
- createdEntityIds.addAll( type2Identities );
-
-
- final TestProgressObserver progressObserver = new TestProgressObserver();
-
-
- //load everything that appears in v1, migrate and ensure it appears in v2
- final Set<MvccEntity> savedEntities = new HashSet<>( 10000 );
- //set that holds all entityIds for later assertion
- final Set<Id> entityIds = new HashSet<>(10000);
-
-
- //read everything in previous version format and put it into our types. Assumes we're
- //using a test system, and it's not a huge amount of data, otherwise we'll overflow.
-
- allEntitiesInSystemObservable.getAllEntitiesInSystem( 1000)
- .doOnNext( new Action1<ApplicationEntityGroup<CollectionScope>>() {
- @Override
- public void call(
- final ApplicationEntityGroup<CollectionScope> entity ) {
-
- //add all versions from history to our comparison
- for ( final EntityIdScope id : entity.entityIds ) {
-
- CollectionScope scope = CpNamingUtils
- .getCollectionScopeNameFromEntityType(
- entity.applicationScope.getApplication(),
- id.getId().getType());
-
- final Iterator<MvccEntity> versions = v1Strategy
- .loadDescendingHistory( scope, id.getId(), UUIDGenerator.newTimeUUID(),
- 100 );
-
- while ( versions.hasNext() ) {
-
- final MvccEntity mvccEntity = versions.next();
-
- savedEntities.add( mvccEntity );
-
- createdEntityIds.remove( mvccEntity.getId() );
-
- entityIds.add( id.getId() );
- }
- }
- }
- } ).toBlocking().lastOrDefault( null );
-
- assertEquals( "Newly saved entities encountered", 0, createdEntityIds.size() );
- assertTrue( "Saved new entities", savedEntities.size() > 0 );
-
- //perform the migration
- rx.Observable<ApplicationEntityGroup> oRx = allEntitiesInSystemObservable.getAllEntitiesInSystem(applicationObservable.getAllApplicationScopes(), 1000);
-
- entityDataMigration.migrate(oRx, progressObserver).toBlocking().last();
-
- assertFalse( "Progress observer should not have failed", progressObserver.getFailed() );
- assertTrue( "Progress observer should have update messages", progressObserver.getUpdates().size() > 0 );
-
-
- //write the status and version, then invalidate the cache so it appears
- migrationInfoSerialization.setStatusCode( DataMigrationManagerImpl.StatusCode.COMPLETE.status );
- migrationInfoSerialization.setVersion( entityDataMigration.getVersion() );
- dataMigrationManager.invalidate();
-
- assertEquals( "New version saved, and we should get new implementation", entityDataMigration.getVersion(),
- dataMigrationManager.getCurrentVersion() );
-
-
- //now visit all entities in the system again, load them from v2, and ensure they're the same
- allEntitiesInSystemObservable.getAllEntitiesInSystem( 1000)
- .doOnNext( new Action1<ApplicationEntityGroup<CollectionScope>>() {
- @Override
- public void call(
- final ApplicationEntityGroup<CollectionScope> entity ) {
- //add all versions from history to our comparison
- for ( final EntityIdScope<CollectionScope> id : entity.entityIds ) {
-
- CollectionScope scope = CpNamingUtils
- .getCollectionScopeNameFromEntityType(
- entity.applicationScope.getApplication(),
- id.getId().getType());
-
- final Iterator<MvccEntity> versions = v2Strategy
- .loadDescendingHistory( scope, id.getId(),
- UUIDGenerator.newTimeUUID(), 100 );
-
- while ( versions.hasNext() ) {
-
- final MvccEntity mvccEntity = versions.next();
-
- savedEntities.remove( mvccEntity );
- }
- }
- }
- }
-
-
- ).toBlocking().lastOrDefault( null );
-
-
- assertEquals( "All entities migrated", 0, savedEntities.size() );
-
-
- //now visit all entities in the system again, and load them from the EM,
- // ensure we see everything we did in the v1 traversal
- allEntitiesInSystemObservable.getAllEntitiesInSystem( 1000)
- .doOnNext( new Action1<ApplicationEntityGroup<CollectionScope>>() {
- @Override
- public void call(
- final ApplicationEntityGroup<CollectionScope> entity ) {
-
- final EntityManager em = emf.getEntityManager(
- entity.applicationScope.getApplication().getUuid() );
-
- //add all versions from history to our comparison
- for ( final EntityIdScope<CollectionScope> id : entity.entityIds ) {
-
-
- try {
- final Entity emEntity = em.get( SimpleEntityRef.fromId( id.getId() ) );
-
- if(emEntity != null){
- entityIds.remove( id.getId() );
- }
- }
- catch ( Exception e ) {
- throw new RuntimeException("Error loading entity", e);
- }
- }
- }
- }
-
-
- ).toBlocking().lastOrDefault( null );
-
-
- assertEquals("All entities could be loaded by the entity manager", 0, entityIds.size());
-
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/802dcdec/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/EntityTypeMappingMigrationIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/EntityTypeMappingMigrationIT.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/EntityTypeMappingMigrationIT.java
index 421b875..99fb139 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/EntityTypeMappingMigrationIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/EntityTypeMappingMigrationIT.java
@@ -36,7 +36,6 @@ import org.apache.usergrid.AbstractCoreIT;
import org.apache.usergrid.cassandra.SpringResource;
import org.apache.usergrid.corepersistence.EntityWriteHelper;
import org.apache.usergrid.corepersistence.ManagerCache;
-import org.apache.usergrid.corepersistence.rx.impl.AllEntitiesInSystemObservableImpl;
import org.apache.usergrid.corepersistence.util.CpNamingUtils;
import org.apache.usergrid.persistence.Entity;
import org.apache.usergrid.persistence.EntityManager;
[30/50] [abbrv] incubator-usergrid git commit: WIP overwrite
Posted by sf...@apache.org.
WIP overwrite
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/ade97149
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/ade97149
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/ade97149
Branch: refs/heads/USERGRID-480
Commit: ade9714955600c9f23198ad18ff5d23d960b216d
Parents: 0c3b439
Author: Todd Nine <tn...@apigee.com>
Authored: Sun Mar 8 16:53:32 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Sun Mar 8 16:53:32 2015 -0600
----------------------------------------------------------------------
.../corepersistence/CpEntityManagerFactory.java | 36 +-------------
.../persistence/EntityManagerFactory.java | 20 --------
.../cassandra/EntityManagerFactoryImpl.java | 23 ---------
.../data/DataMigrationManagerImplTest.java | 34 +++++++++++--
.../apache/usergrid/rest/MigrateResource.java | 52 +++++++++++++-------
5 files changed, 63 insertions(+), 102 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ade97149/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index 3b39bc9..8a1b0c4 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -109,7 +109,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
private ManagerCache managerCache;
- private DataMigrationManager dataMigrationManager;
+
private CassandraService cassandraService;
private CounterUtils counterUtils;
@@ -123,9 +123,6 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
this.counterUtils = counterUtils;
this.injector = injector;
this.managerCache = injector.getInstance( ManagerCache.class );
- this.dataMigrationManager = injector.getInstance( DataMigrationManager.class );
-
-
}
@@ -165,8 +162,6 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
if ( managerCache == null ) {
managerCache = injector.getInstance( ManagerCache.class );
-
- dataMigrationManager = injector.getInstance( DataMigrationManager.class );
}
return managerCache;
}
@@ -779,35 +774,6 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
}
- @Override
- public void migrateData() throws Exception {
- dataMigrationManager.migrate();
- }
-
-
- @Override
- public String getMigrateDataStatus() {
- //TODO USERGRID-405 return dataMigrationManager.getLastStatus();
- return null;
- }
-
-
- @Override
- public int getMigrateDataVersion() {
- //TODO USERGRID-405
- //return dataMigrationManager.getCurrentVersion();
- return 0;
- }
-
-
- @Override
- public void setMigrationVersion( final int version ) {
- //TODO USERGRID-405
-//
-// dataMigrationManager.resetToVersion( version );
-// dataMigrationManager.invalidate();
- }
-
@Override
public void flushEntityManagerCaches() {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ade97149/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
index 880fe89..f706ac1 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
@@ -127,26 +127,6 @@ public interface EntityManagerFactory {
public void rebuildApplicationIndexes( UUID appId, ProgressObserver po ) throws Exception;
- /**
- * Perform any data migrations necessary in the system
- * @throws Exception
- */
- public void migrateData() throws Exception;
-
- /**
- * Return the migration status message
- */
- public String getMigrateDataStatus();
-
- /**
- * Return the current migration version of the system
- */
- public int getMigrateDataVersion();
-
- /**
- * Force the migration version to the specified version
- */
- public void setMigrationVersion(int version);
/**
* Perform a realtime count of every entity in the system. This can be slow as it traverses the entire system graph
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ade97149/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
index af1eabc..7b55e5b 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
@@ -443,29 +443,6 @@ public class EntityManagerFactoryImpl implements EntityManagerFactory, Applicati
@Override
- public void migrateData() throws Exception {
-
- }
-
-
- @Override
- public String getMigrateDataStatus() {
- throw new UnsupportedOperationException("Not supported in v1");
- }
-
-
- @Override
- public int getMigrateDataVersion() {
- throw new UnsupportedOperationException("Not supported in v1");
- }
-
-
- @Override
- public void setMigrationVersion( final int version ) {
- throw new UnsupportedOperationException("Not supported in v1");
- }
-
- @Override
public void addIndex(UUID appId, String suffix,final int shards,final int replicas,final String consistency) {
throw new UnsupportedOperationException("Not supported in v1");
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ade97149/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManagerImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManagerImplTest.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManagerImplTest.java
index 319282c..c2eef3f 100644
--- a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManagerImplTest.java
+++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManagerImplTest.java
@@ -299,12 +299,36 @@ public class DataMigrationManagerImplTest {
@Test
public void testLastStatus() {
- fail( "writeme" );
- }
+ final String name = "plugin1";
+ final String status = "some status";
- @Test
- public void testLastStatusNoPlugin() {
- fail( "writeme" );
+ //linked hash set is intentional here. For iteration order we can boostrap to come second so we can
+ //verify it was actually run first
+ final Set<MigrationPlugin> plugins = new LinkedHashSet<>();
+
+ MigrationPlugin plugin1 = mock( MigrationPlugin.class );
+ when( plugin1.getPhase() ).thenReturn( PluginPhase.MIGRATE );
+
+
+ when( plugin1.getName() ).thenReturn( name );
+
+
+
+ plugins.add( plugin1 );
+
+
+ final MigrationInfoSerialization migrationInfoSerialization = mock( MigrationInfoSerialization.class );
+ when(migrationInfoSerialization.getStatusMessage( name )).thenReturn( status );
+
+
+ DataMigrationManagerImpl migrationManager = new DataMigrationManagerImpl( plugins, migrationInfoSerialization );
+
+ final String returnedStatus = migrationManager.getLastStatus( name );
+
+
+ assertEquals(status, returnedStatus);
}
+
+
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ade97149/stack/rest/src/main/java/org/apache/usergrid/rest/MigrateResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/MigrateResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/MigrateResource.java
index 9ab5dc3..702d134 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/MigrateResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/MigrateResource.java
@@ -33,22 +33,25 @@ import javax.ws.rs.core.UriInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component;
+import org.apache.usergrid.persistence.core.migration.schema.MigrationManager;
import org.apache.usergrid.rest.security.annotations.RequireSystemAccess;
import com.fasterxml.jackson.databind.node.JsonNodeFactory;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.common.base.Preconditions;
+import com.google.inject.Injector;
import com.sun.jersey.api.json.JSONWithPadding;
@Component
@Scope( "singleton" )
@Produces( {
- MediaType.APPLICATION_JSON, "application/javascript", "application/x-javascript", "text/ecmascript",
- "application/ecmascript", "text/jscript"
+ MediaType.APPLICATION_JSON, "application/javascript", "application/x-javascript", "text/ecmascript",
+ "application/ecmascript", "text/jscript"
} )
public class MigrateResource extends AbstractContextResource {
@@ -59,6 +62,8 @@ public class MigrateResource extends AbstractContextResource {
logger.info( "SystemResource initialized" );
}
+ @Autowired
+ private Injector guiceInjector;
@RequireSystemAccess
@@ -66,13 +71,15 @@ public class MigrateResource extends AbstractContextResource {
@Path( "run" )
public JSONWithPadding migrateData( @Context UriInfo ui,
@QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
- throws Exception {
+ throws Exception {
ApiResponse response = createApiResponse();
response.setAction( "Migrate Data" );
//TODO make this use the task scheduler
+
+
final Thread migrate = new Thread() {
@Override
@@ -80,7 +87,7 @@ public class MigrateResource extends AbstractContextResource {
logger.info( "Migrating Data " );
try {
- emf.migrateData();
+ getMigrationManager().migrate();
}
catch ( Exception e ) {
logger.error( "Unable to migrate data", e );
@@ -99,21 +106,24 @@ public class MigrateResource extends AbstractContextResource {
}
+
@RequireSystemAccess
@PUT
@Path( "set" )
public JSONWithPadding setMigrationVersion( @Context UriInfo ui, Map<String, Object> json,
@QueryParam( "callback" ) @DefaultValue( "" ) String callback )
- throws Exception {
+ throws Exception {
logger.debug( "newOrganization" );
Preconditions.checkNotNull( json, "You must provide a json body" );
+ Preconditions.checke
+
String version = ( String ) json.get( "version" );
- Preconditions.checkArgument( version != null && version.length() > 0,
- "You must specify a version field in your json" );
+ Preconditions
+ .checkArgument( version != null && version.length() > 0, "You must specify a version field in your json" );
int intVersion = Integer.parseInt( version );
@@ -130,7 +140,7 @@ public class MigrateResource extends AbstractContextResource {
@Path( "status" )
public JSONWithPadding migrateStatus( @Context UriInfo ui,
@QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
- throws Exception {
+ throws Exception {
ApiResponse response = createApiResponse();
response.setAction( "Migrate Schema indexes" );
@@ -145,21 +155,25 @@ public class MigrateResource extends AbstractContextResource {
return new JSONWithPadding( response, callback );
}
+
@RequireSystemAccess
- @GET
- @Path( "count" )
- public JSONWithPadding migrateCount( @Context UriInfo ui,
- @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
- throws Exception {
+ @GET
+ @Path( "count" )
+ public JSONWithPadding migrateCount( @Context UriInfo ui,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+ throws Exception {
- ApiResponse response = createApiResponse();
- response.setAction( "Current entity count in system" );
+ ApiResponse response = createApiResponse();
+ response.setAction( "Current entity count in system" );
- response.setProperty( "count", emf.performEntityCount() );
+ response.setProperty( "count", emf.performEntityCount() );
- response.setSuccess();
+ response.setSuccess();
- return new JSONWithPadding( response, callback );
- }
+ return new JSONWithPadding( response, callback );
+ }
+ private MigrationManager getMigrationManager(){
+ return guiceInjector.getInstance( MigrationManager.class );
+ }
}
[09/50] [abbrv] incubator-usergrid git commit: Core tests passing.
Posted by sf...@apache.org.
Core tests passing.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/7925a4b8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/7925a4b8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/7925a4b8
Branch: refs/heads/USERGRID-480
Commit: 7925a4b8353c9bded60188685a2109cba5024faa
Parents: fa69be8
Author: Todd Nine <tn...@apigee.com>
Authored: Tue Mar 3 13:53:18 2015 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Tue Mar 3 13:53:18 2015 -0700
----------------------------------------------------------------------
...MvccEntitySerializationStrategyImplTest.java | 4 +-
...ntitySerializationStrategyProxyV1_3Test.java | 3 -
...ccEntitySerializationStrategyV1ImplTest.java | 71 +++++++++++++
...ccEntitySerializationStrategyV2ImplTest.java | 67 ++++++++++++
...ccEntitySerializationStrategyV3ImplTest.java | 9 +-
.../MvccEntitySerializationStrategyV3Test.java | 103 -------------------
.../data/newimpls/AbstractMigrationPlugin.java | 1 +
.../persistence/graph/guice/GraphModule.java | 14 ++-
.../impl/EdgeDataMigrationImpl.java | 79 +++++++-------
.../EdgeMetadataSerializationProxyImpl.java | 13 +--
.../graph/guice/TestGraphModule.java | 35 +++----
.../EdgeMetaDataSerializationProxyV1Test.java | 12 ++-
.../EdgeMetaDataSerializationProxyV2Test.java | 21 ++--
.../EdgeMetaDataSerializationV1Test.java | 7 +-
.../EdgeMetaDataSerializationV2Test.java | 7 +-
.../persistence/map/guice/TestMapModule.java | 9 +-
.../index/guice/TestIndexModule.java | 10 +-
.../queue/guice/TestQueueModule.java | 9 +-
18 files changed, 252 insertions(+), 222 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7925a4b8/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImplTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImplTest.java
index ac67558..f76230d 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImplTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImplTest.java
@@ -423,7 +423,7 @@ public abstract class MvccEntitySerializationStrategyImplTest {
serializationStrategy.delete( context, id, version3 ).execute();
- assertFalse("Not loaded", serializationStrategy.load( context, id ).isPresent());
+ assertFalse( "Not loaded", serializationStrategy.load( context, id ).isPresent() );
}
@@ -465,7 +465,7 @@ public abstract class MvccEntitySerializationStrategyImplTest {
/**
- * We no longer support partial writes, ensure that an exception is thrown when this occurs
+ * We no longer support partial writes, ensure that an exception is thrown when this occurs after v3
* @throws ConnectionException
*/
@Test(expected = UnsupportedOperationException.class)
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7925a4b8/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV1_3Test.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV1_3Test.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV1_3Test.java
index 81113b1..5c8a61f 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV1_3Test.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV1_3Test.java
@@ -47,9 +47,6 @@ public class MvccEntitySerializationStrategyProxyV1_3Test extends MvccEntitySeri
@Inject
private MvccEntitySerializationStrategyV1Impl mvccEntitySerializationStrategyV1;
- @Inject
- private VersionedMigrationSet<MvccEntitySerializationStrategy> migrationSet;
-
@Override
protected MvccEntitySerializationStrategy getMvccEntitySerializationStrategy() {
return serializationStrategy;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7925a4b8/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV1ImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV1ImplTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV1ImplTest.java
index d4d6fb5..4e0c8cc 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV1ImplTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV1ImplTest.java
@@ -20,14 +20,34 @@
package org.apache.usergrid.persistence.collection.serialization.impl;
+import java.util.UUID;
+
+import org.junit.Test;
import org.junit.runner.RunWith;
+import org.apache.usergrid.persistence.collection.CollectionScope;
+import org.apache.usergrid.persistence.collection.MvccEntity;
import org.apache.usergrid.persistence.collection.guice.TestCollectionModule;
+import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl;
+import org.apache.usergrid.persistence.collection.mvcc.entity.impl.MvccEntityImpl;
import org.apache.usergrid.persistence.collection.serialization.MvccEntitySerializationStrategy;
+import org.apache.usergrid.persistence.collection.util.EntityUtils;
import org.apache.usergrid.persistence.core.test.ITRunner;
import org.apache.usergrid.persistence.core.test.UseModules;
+import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.entity.Id;
+import org.apache.usergrid.persistence.model.entity.SimpleId;
+import org.apache.usergrid.persistence.model.field.BooleanField;
+import org.apache.usergrid.persistence.model.field.DoubleField;
+import org.apache.usergrid.persistence.model.field.IntegerField;
+import org.apache.usergrid.persistence.model.field.LongField;
+import org.apache.usergrid.persistence.model.field.StringField;
+import org.apache.usergrid.persistence.model.field.UUIDField;
+import org.apache.usergrid.persistence.model.util.UUIDGenerator;
+import com.google.common.base.Optional;
import com.google.inject.Inject;
+import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
@RunWith( ITRunner.class )
@@ -43,4 +63,55 @@ public class MvccEntitySerializationStrategyV1ImplTest extends MvccEntitySeriali
protected MvccEntitySerializationStrategy getMvccEntitySerializationStrategy() {
return serializationStrategy;
}
+
+
+
+ /**
+ * We no longer support partial writes, ensure that an exception is thrown when this occurs after v3
+ * @throws com.netflix.astyanax.connectionpool.exceptions.ConnectionException
+ */
+ @Test
+ public void writeLoadDeletePartial() throws ConnectionException {
+
+ final Id organizationId = new SimpleId( "organization" );
+ final Id applicationId = new SimpleId( "application" );
+ final String name = "test";
+
+ CollectionScope context = new CollectionScopeImpl( organizationId, applicationId, name );
+
+
+ final UUID entityId = UUIDGenerator.newTimeUUID();
+ final UUID version = UUIDGenerator.newTimeUUID();
+ final String type = "test";
+
+ final Id id = new SimpleId( entityId, type );
+
+ Entity entity = new Entity( id );
+
+ EntityUtils.setVersion( entity, version );
+
+
+ BooleanField boolField = new BooleanField( "boolean", false );
+ DoubleField doubleField = new DoubleField( "double", 1d );
+ IntegerField intField = new IntegerField( "long", 1 );
+ LongField longField = new LongField( "int", 1l );
+ StringField stringField = new StringField( "name", "test" );
+ UUIDField uuidField = new UUIDField( "uuid", UUIDGenerator.newTimeUUID() );
+
+ entity.setField( boolField );
+ entity.setField( doubleField );
+ entity.setField( intField );
+ entity.setField( longField );
+ entity.setField( stringField );
+ entity.setField( uuidField );
+
+
+ MvccEntity saved = new MvccEntityImpl( id, version, MvccEntity.Status.PARTIAL, Optional.of( entity ) );
+
+
+ //persist the entity
+ serializationStrategy.write( context, saved ).execute();
+
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7925a4b8/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2ImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2ImplTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2ImplTest.java
index 97bcb1f..b91c453 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2ImplTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2ImplTest.java
@@ -20,14 +20,34 @@
package org.apache.usergrid.persistence.collection.serialization.impl;
+import java.util.UUID;
+
+import org.junit.Test;
import org.junit.runner.RunWith;
+import org.apache.usergrid.persistence.collection.CollectionScope;
+import org.apache.usergrid.persistence.collection.MvccEntity;
import org.apache.usergrid.persistence.collection.guice.TestCollectionModule;
+import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl;
+import org.apache.usergrid.persistence.collection.mvcc.entity.impl.MvccEntityImpl;
import org.apache.usergrid.persistence.collection.serialization.MvccEntitySerializationStrategy;
+import org.apache.usergrid.persistence.collection.util.EntityUtils;
import org.apache.usergrid.persistence.core.test.ITRunner;
import org.apache.usergrid.persistence.core.test.UseModules;
+import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.entity.Id;
+import org.apache.usergrid.persistence.model.entity.SimpleId;
+import org.apache.usergrid.persistence.model.field.BooleanField;
+import org.apache.usergrid.persistence.model.field.DoubleField;
+import org.apache.usergrid.persistence.model.field.IntegerField;
+import org.apache.usergrid.persistence.model.field.LongField;
+import org.apache.usergrid.persistence.model.field.StringField;
+import org.apache.usergrid.persistence.model.field.UUIDField;
+import org.apache.usergrid.persistence.model.util.UUIDGenerator;
+import com.google.common.base.Optional;
import com.google.inject.Inject;
+import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
@RunWith( ITRunner.class )
@@ -43,5 +63,52 @@ public class MvccEntitySerializationStrategyV2ImplTest extends MvccEntitySeriali
return serializationStrategy;
}
+ /**
+ * We no longer support partial writes, ensure that an exception is thrown when this occurs after v3
+ * @throws com.netflix.astyanax.connectionpool.exceptions.ConnectionException
+ */
+ @Test
+ public void writeLoadDeletePartial() throws ConnectionException {
+
+ final Id organizationId = new SimpleId( "organization" );
+ final Id applicationId = new SimpleId( "application" );
+ final String name = "test";
+
+ CollectionScope context = new CollectionScopeImpl( organizationId, applicationId, name );
+
+
+ final UUID entityId = UUIDGenerator.newTimeUUID();
+ final UUID version = UUIDGenerator.newTimeUUID();
+ final String type = "test";
+
+ final Id id = new SimpleId( entityId, type );
+
+ Entity entity = new Entity( id );
+
+ EntityUtils.setVersion( entity, version );
+
+
+ BooleanField boolField = new BooleanField( "boolean", false );
+ DoubleField doubleField = new DoubleField( "double", 1d );
+ IntegerField intField = new IntegerField( "long", 1 );
+ LongField longField = new LongField( "int", 1l );
+ StringField stringField = new StringField( "name", "test" );
+ UUIDField uuidField = new UUIDField( "uuid", UUIDGenerator.newTimeUUID() );
+
+ entity.setField( boolField );
+ entity.setField( doubleField );
+ entity.setField( intField );
+ entity.setField( longField );
+ entity.setField( stringField );
+ entity.setField( uuidField );
+
+
+ MvccEntity saved = new MvccEntityImpl( id, version, MvccEntity.Status.PARTIAL, Optional.of( entity ) );
+
+
+ //persist the entity
+ serializationStrategy.write( context, saved ).execute();
+
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7925a4b8/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3ImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3ImplTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3ImplTest.java
index b30e901..e5ea409 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3ImplTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3ImplTest.java
@@ -21,13 +21,11 @@ import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
/**
- * Classy class class.
+ * Same tests as v2, we just override some methods to ensure they throw the correct exceptions
*/
-
-
@RunWith( ITRunner.class )
@UseModules( TestCollectionModule.class )
-public class MvccEntitySerializationStrategyV3ImplTest extends MvccEntitySerializationStrategyV3Test {
+public class MvccEntitySerializationStrategyV3ImplTest extends MvccEntitySerializationStrategyV2Test {
@Inject
private MvccEntitySerializationStrategyV3Impl serializationStrategy;
@@ -70,4 +68,7 @@ public class MvccEntitySerializationStrategyV3ImplTest extends MvccEntitySeriali
serializationStrategy.loadDescendingHistory( context, entityId, version1, 20 );
}
+
+
}
+
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7925a4b8/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3Test.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3Test.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3Test.java
deleted file mode 100644
index 137bb1f..0000000
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3Test.java
+++ /dev/null
@@ -1,103 +0,0 @@
-package org.apache.usergrid.persistence.collection.serialization.impl;
-
-import com.google.inject.Inject;
-import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
-import org.apache.usergrid.persistence.collection.CollectionScope;
-import org.apache.usergrid.persistence.collection.MvccEntity;
-import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl;
-import org.apache.usergrid.persistence.collection.mvcc.entity.impl.MvccEntityImpl;
-import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
-import org.apache.usergrid.persistence.collection.util.EntityHelper;
-import org.apache.usergrid.persistence.core.astyanax.CassandraFig;
-import org.apache.usergrid.persistence.core.guicyfig.SetConfigTestBypass;
-import org.apache.usergrid.persistence.core.util.ValidationUtils;
-import org.apache.usergrid.persistence.model.entity.Entity;
-import org.apache.usergrid.persistence.model.entity.Id;
-import org.apache.usergrid.persistence.model.entity.SimpleId;
-import org.apache.usergrid.persistence.model.util.UUIDGenerator;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.UUID;
-
-import static org.junit.Assert.assertTrue;
-
-/**
- * Classy class class.
- */
-public abstract class MvccEntitySerializationStrategyV3Test extends MvccEntitySerializationStrategyImplTest{
- @Inject
- protected SerializationFig serializationFig;
-
- @Inject
- protected CassandraFig cassandraFig;
-
- private int setMaxEntitySize;
-
- @Before
- public void setUp() {
- setMaxEntitySize = serializationFig.getMaxEntitySize();
- }
-
-
- @After
- public void tearDown() {
- SetConfigTestBypass.setValueByPass(serializationFig, "getMaxEntitySize", setMaxEntitySize + "");
- }
-
- /**
- * Tests an entity with more than 65535 bytes worth of data is successfully stored and retrieved
- */
- @Test
- public void largeEntityWriteRead() throws ConnectionException {
- final int setSize = 65535 * 2;
-
-
- //this is the size it works out to be when serialized, we want to allow this size
-
- SetConfigTestBypass.setValueByPass( serializationFig, "getMaxEntitySize", 65535 * 10 + "" );
- final Entity entity = EntityHelper.generateEntity(setSize);
-
- //now we have one massive, entity, save it and retrieve it.
- CollectionScope context =
- new CollectionScopeImpl( new SimpleId( "organization" ), new SimpleId( "parent" ), "tests" );
-
-
- final Id id = entity.getId();
- ValidationUtils.verifyIdentity(id);
- final UUID version = UUIDGenerator.newTimeUUID();
- final MvccEntity.Status status = MvccEntity.Status.COMPLETE;
-
- final MvccEntity mvccEntity = new MvccEntityImpl( id, version, status, entity );
-
-
- getMvccEntitySerializationStrategy().write( context, mvccEntity ).execute();
-
- //now load it
- final MvccEntity loadedEntity =
- getMvccEntitySerializationStrategy().load( context, id ).get();
-
-
- assertLargeEntity( mvccEntity, loadedEntity );
-
-
- MvccEntity returned =
- serializationStrategy.load( context, Collections.singleton(id), version ).getEntity( id );
-
- assertLargeEntity( mvccEntity, returned );
- }
-
-
-
- protected void assertLargeEntity( final MvccEntity expected, final MvccEntity returned ) {
-
- org.junit.Assert.assertEquals( "The loaded entity should match the stored entity", expected, returned );
-
- EntityHelper.verifyDeepEquals( expected.getEntity().get(), returned.getEntity().get() );
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7925a4b8/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/AbstractMigrationPlugin.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/AbstractMigrationPlugin.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/AbstractMigrationPlugin.java
index 7b3aa00..17d88d0 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/AbstractMigrationPlugin.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/AbstractMigrationPlugin.java
@@ -115,6 +115,7 @@ public abstract class AbstractMigrationPlugin<T> implements MigrationPlugin {
//run the migration
final int newSystemVersion = migrationToExecute.migrate( version, entityIdScopeDataMigrationProvider, po );
+ //write the version
migrationInfoSerialization.setVersion( getName(), newSystemVersion );
//signal we've run a migration and return
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7925a4b8/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/guice/GraphModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/guice/GraphModule.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/guice/GraphModule.java
index 84fbaac..67459b2 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/guice/GraphModule.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/guice/GraphModule.java
@@ -85,7 +85,7 @@ import com.google.inject.TypeLiteral;
import com.google.inject.multibindings.Multibinder;
-public class GraphModule extends AbstractModule {
+public abstract class GraphModule extends AbstractModule {
@Override
protected void configure() {
@@ -190,6 +190,9 @@ public class GraphModule extends AbstractModule {
bind( EdgeMetadataSerializationV2Impl.class );
bind(EdgeMetadataSerialization.class).annotatedWith( ProxyImpl.class ).to( EdgeMetadataSerializationProxyImpl.class );
+ //invoke the migration plugin config
+ configureMigrationProvider();
+
}
@@ -232,6 +235,15 @@ public class GraphModule extends AbstractModule {
}
+ /**
+ * Gives callers the ability to to configure an instance of
+ *
+ * MigrationDataProvider<ApplicationScope> for providing data migrations
+ */
+ public abstract void configureMigrationProvider();
+
+
+
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7925a4b8/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeDataMigrationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeDataMigrationImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeDataMigrationImpl.java
index 862c796..49a954d 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeDataMigrationImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeDataMigrationImpl.java
@@ -39,6 +39,7 @@ import org.slf4j.LoggerFactory;
import rx.Observable;
import rx.functions.Action1;
import rx.functions.Func1;
+import rx.functions.Func2;
import java.util.List;
import java.util.concurrent.atomic.AtomicLong;
@@ -82,53 +83,53 @@ public class EdgeDataMigrationImpl implements DataMigration2<ApplicationScope> {
final MigrationRelationship<EdgeMetadataSerialization>
migration = allVersions.getMigrationRelationship( currentVersion );
- migrationDataProvider.getData().flatMap(new Func1<ApplicationScope, Observable<?>>() {
+ final Observable<List<Edge>> observable = migrationDataProvider.getData().flatMap(new Func1<ApplicationScope, Observable<List<Edge>>>() {
@Override
- public Observable call(final ApplicationScope applicationScope) {
- final GraphManager gm = graphManagerFactory.createEdgeManager(applicationScope);
- final Observable<Edge> edgesFromSource = edgesFromSourceObservable.edgesFromSource(gm, applicationScope.getApplication());
- logger.info("Migrating edges scope {}", applicationScope);
+ public Observable<List<Edge>> call(final ApplicationScope applicationScope) {
+ final GraphManager gm = graphManagerFactory.createEdgeManager( applicationScope );
+ final Observable<Edge> edgesFromSource =
+ edgesFromSourceObservable.edgesFromSource( gm, applicationScope.getApplication() );
+ logger.info( "Migrating edges scope {}", applicationScope );
//get each edge from this node as a source
return edgesFromSource
- //for each edge, re-index it in v2 every 1000 edges or less
- .buffer( 1000 )
- //do the writes of 1k in parallel
- .parallel( new Func1<Observable<List<Edge>>, Observable>() {
+ //for each edge, re-index it in v2 every 1000 edges or less
+ .buffer( 1000 ).parallel( new Func1<Observable<List<Edge>>, Observable<List<Edge>>>() {
@Override
- public Observable call( final Observable<List<Edge>> listObservable ) {
+ public Observable<List<Edge>> call( final Observable<List<Edge>> listObservable ) {
return listObservable.doOnNext( new Action1<List<Edge>>() {
- @Override
- public void call( List<Edge> edges ) {
- final MutationBatch batch = keyspace.prepareMutationBatch();
-
- for ( Edge edge : edges ) {
- logger.info( "Migrating meta for edge {}", edge );
- final MutationBatch edgeBatch =
- migration.to.writeEdge( applicationScope, edge );
- batch.mergeShallow( edgeBatch );
- }
-
- try {
- batch.execute();
- }
- catch ( ConnectionException e ) {
- throw new RuntimeException( "Unable to perform migration", e );
- }
-
- //update the observer so the admin can see it
- final long newCount = counter.addAndGet( edges.size() );
-
- observer.update( migration.to.getImplementationVersion(),
- String.format( "Currently running. Rewritten %d edge types",
- newCount ) );
- }
- } );
+ @Override
+ public void call( List<Edge> edges ) {
+ final MutationBatch batch = keyspace.prepareMutationBatch();
+
+ for ( Edge edge : edges ) {
+ logger.info( "Migrating meta for edge {}", edge );
+ final MutationBatch edgeBatch =
+ migration.to.writeEdge( applicationScope, edge );
+ batch.mergeShallow( edgeBatch );
+ }
+
+ try {
+ batch.execute();
+ }
+ catch ( ConnectionException e ) {
+ throw new RuntimeException( "Unable to perform migration", e );
+ }
+
+ //update the observer so the admin can see it
+ final long newCount = counter.addAndGet( edges.size() );
+
+ observer.update( migration.to.getImplementationVersion(),
+ String.format( "Currently running. Rewritten %d edge types",
+ newCount ) );
+ }
+ } );
}
} );
- }
- });
+ }});
+
+ observable.longCount().toBlocking().last();
return migration.to.getImplementationVersion();
@@ -139,7 +140,7 @@ public class EdgeDataMigrationImpl implements DataMigration2<ApplicationScope> {
@Override
public boolean supports( final int currentVersion ) {
- return currentVersion <= edgeMetadataSerializationV2.getImplementationVersion();
+ return currentVersion < edgeMetadataSerializationV2.getImplementationVersion();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7925a4b8/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeMetadataSerializationProxyImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeMetadataSerializationProxyImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeMetadataSerializationProxyImpl.java
index a5bde21..262d665 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeMetadataSerializationProxyImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeMetadataSerializationProxyImpl.java
@@ -26,15 +26,18 @@ import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import org.apache.usergrid.persistence.core.astyanax.MultiTennantColumnFamilyDefinition;
-import org.apache.usergrid.persistence.core.migration.data.DataMigrationManager;
import org.apache.usergrid.persistence.core.migration.data.MigrationInfoCache;
import org.apache.usergrid.persistence.core.migration.data.newimpls.MigrationRelationship;
import org.apache.usergrid.persistence.core.migration.data.newimpls.VersionedMigrationSet;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.graph.*;
+import org.apache.usergrid.persistence.graph.Edge;
+import org.apache.usergrid.persistence.graph.SearchEdgeType;
+import org.apache.usergrid.persistence.graph.SearchIdType;
import org.apache.usergrid.persistence.graph.serialization.EdgeMetadataSerialization;
-import org.apache.usergrid.persistence.graph.serialization.EdgeMigrationStrategy;
import org.apache.usergrid.persistence.graph.serialization.impl.migration.GraphMigrationPlugin;
import org.apache.usergrid.persistence.model.entity.Id;
@@ -42,12 +45,10 @@ import com.google.inject.Inject;
import com.google.inject.Singleton;
import com.netflix.astyanax.Keyspace;
import com.netflix.astyanax.MutationBatch;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
@Singleton
-public class EdgeMetadataSerializationProxyImpl implements EdgeMetadataSerialization, EdgeMigrationStrategy {
+public class EdgeMetadataSerializationProxyImpl implements EdgeMetadataSerialization {
private static final Logger logger = LoggerFactory.getLogger(EdgeMetadataSerializationProxyImpl.class);
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7925a4b8/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/guice/TestGraphModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/guice/TestGraphModule.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/guice/TestGraphModule.java
index 63cb204..7dd2254 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/guice/TestGraphModule.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/guice/TestGraphModule.java
@@ -20,12 +20,12 @@ package org.apache.usergrid.persistence.graph.guice;
import org.apache.usergrid.persistence.core.guice.CommonModule;
-import org.apache.usergrid.persistence.core.guice.MaxMigrationModule;
import org.apache.usergrid.persistence.core.guice.TestModule;
-import org.apache.usergrid.persistence.core.rx.AllEntitiesInSystemObservable;
-import org.apache.usergrid.persistence.core.rx.AllEntitiesInSystemTestObservable;
-import org.apache.usergrid.persistence.core.rx.ApplicationObservable;
-import org.apache.usergrid.persistence.core.rx.ApplicationsTestObservable;
+import org.apache.usergrid.persistence.core.migration.data.newimpls.MigrationDataProvider;
+import org.apache.usergrid.persistence.core.migration.data.newimpls.TestMigrationDataProvider;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+
+import com.google.inject.TypeLiteral;
/**
@@ -38,18 +38,17 @@ public class TestGraphModule extends TestModule {
/**
* Runtime modules
*/
- bind(ApplicationObservable.class).to(ApplicationsTestObservable.class);
-
- bind(AllEntitiesInSystemObservable.class).to(AllEntitiesInSystemTestObservable.class);
-
- install( new CommonModule());
- install( new GraphModule() );
-
-
- /**
- * Test modules
- */
- install(new MaxMigrationModule());
-
+ install( new CommonModule() );
+ install( new GraphModule() {
+
+ @Override
+ public void configureMigrationProvider() {
+ //configure our migration data provider
+
+ TestMigrationDataProvider<ApplicationScope> migrationDataProvider = new TestMigrationDataProvider<>();
+ bind( new TypeLiteral<MigrationDataProvider<ApplicationScope>>() {} )
+ .toInstance( migrationDataProvider );
+ }
+ } );
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7925a4b8/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/EdgeMetaDataSerializationProxyV1Test.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/EdgeMetaDataSerializationProxyV1Test.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/EdgeMetaDataSerializationProxyV1Test.java
index d6f6943..3c1b4c4 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/EdgeMetaDataSerializationProxyV1Test.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/EdgeMetaDataSerializationProxyV1Test.java
@@ -32,6 +32,9 @@ import org.apache.usergrid.persistence.core.test.ITRunner;
import org.apache.usergrid.persistence.core.test.UseModules;
import org.apache.usergrid.persistence.graph.guice.TestGraphModule;
import org.apache.usergrid.persistence.graph.serialization.impl.EdgeMetadataSerializationProxyImpl;
+import org.apache.usergrid.persistence.graph.serialization.impl.EdgeMetadataSerializationV1Impl;
+import org.apache.usergrid.persistence.graph.serialization.impl.GraphDataVersions;
+import org.apache.usergrid.persistence.graph.serialization.impl.migration.GraphMigrationPlugin;
import com.google.inject.Inject;
@@ -51,6 +54,9 @@ public class EdgeMetaDataSerializationProxyV1Test extends EdgeMetadataSerializat
protected EdgeMetadataSerialization serialization;
@Inject
+ protected EdgeMetadataSerializationV1Impl edgeMetadataSerializationV1;
+
+ @Inject
protected MigrationInfoSerialization migrationInfoSerialization;
private int existingVersion;
@@ -62,16 +68,16 @@ public class EdgeMetaDataSerializationProxyV1Test extends EdgeMetadataSerializat
*/
@Before
public void setMigrationVersion() {
- existingVersion = migrationInfoSerialization.getVersion();
+ existingVersion = migrationInfoSerialization.getVersion( GraphMigrationPlugin.PLUGIN_NAME);
//set our version to 0 so it uses both impls of the proxy
- migrationInfoSerialization.setVersion( 0 );
+ migrationInfoSerialization.setVersion(GraphMigrationPlugin.PLUGIN_NAME, edgeMetadataSerializationV1.getImplementationVersion() );
}
@After
public void reSetMigrationVersion() {
- migrationInfoSerialization.setVersion( existingVersion );
+ migrationInfoSerialization.setVersion( GraphMigrationPlugin.PLUGIN_NAME, existingVersion );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7925a4b8/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/EdgeMetaDataSerializationProxyV2Test.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/EdgeMetaDataSerializationProxyV2Test.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/EdgeMetaDataSerializationProxyV2Test.java
index 69fb72d..35a304d 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/EdgeMetaDataSerializationProxyV2Test.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/EdgeMetaDataSerializationProxyV2Test.java
@@ -32,6 +32,8 @@ import org.apache.usergrid.persistence.core.test.ITRunner;
import org.apache.usergrid.persistence.core.test.UseModules;
import org.apache.usergrid.persistence.graph.guice.TestGraphModule;
import org.apache.usergrid.persistence.graph.serialization.impl.EdgeMetadataSerializationProxyImpl;
+import org.apache.usergrid.persistence.graph.serialization.impl.EdgeMetadataSerializationV2Impl;
+import org.apache.usergrid.persistence.graph.serialization.impl.migration.GraphMigrationPlugin;
import com.google.inject.Inject;
@@ -51,6 +53,10 @@ public class EdgeMetaDataSerializationProxyV2Test extends EdgeMetadataSerializat
protected EdgeMetadataSerialization serialization;
@Inject
+ protected EdgeMetadataSerializationV2Impl edgeMetadataSerializationV2;
+
+
+ @Inject
protected MigrationInfoSerialization migrationInfoSerialization;
private int existingVersion;
@@ -61,21 +67,24 @@ public class EdgeMetaDataSerializationProxyV2Test extends EdgeMetadataSerializat
* correctly
*/
@Before
- public void setMigrationVersion(){
- existingVersion = migrationInfoSerialization.getVersion();
+ public void setMigrationVersion() {
+ existingVersion = migrationInfoSerialization.getVersion( GraphMigrationPlugin.PLUGIN_NAME );
//set our version equal to the new version so it only delegates to the new version
- migrationInfoSerialization.setVersion( EdgeMetadataSerializationProxyImpl.MIGRATION_VERSION );
+ migrationInfoSerialization
+ .setVersion( GraphMigrationPlugin.PLUGIN_NAME, edgeMetadataSerializationV2.getImplementationVersion() );
}
+
@After
- public void reSetMigrationVersion(){
- migrationInfoSerialization.setVersion( existingVersion );
+ public void reSetMigrationVersion() {
+ migrationInfoSerialization.setVersion(GraphMigrationPlugin.PLUGIN_NAME, existingVersion );
}
+
@Override
protected EdgeMetadataSerialization getSerializationImpl() {
- assertTrue(serialization instanceof EdgeMetadataSerializationProxyImpl);
+ assertTrue( serialization instanceof EdgeMetadataSerializationProxyImpl );
return serialization;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7925a4b8/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/EdgeMetaDataSerializationV1Test.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/EdgeMetaDataSerializationV1Test.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/EdgeMetaDataSerializationV1Test.java
index 3fcff68..68a6953 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/EdgeMetaDataSerializationV1Test.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/EdgeMetaDataSerializationV1Test.java
@@ -39,7 +39,6 @@ package org.apache.usergrid.persistence.graph.serialization;/*
*/
-import org.apache.usergrid.persistence.core.guice.V1Impl;
import org.junit.runner.RunWith;
import org.apache.usergrid.persistence.core.test.ITRunner;
@@ -49,21 +48,17 @@ import org.apache.usergrid.persistence.graph.serialization.impl.EdgeMetadataSeri
import com.google.inject.Inject;
-import static org.junit.Assert.assertTrue;
-
@RunWith( ITRunner.class )
@UseModules( { TestGraphModule.class } )
public class EdgeMetaDataSerializationV1Test extends EdgeMetadataSerializationTest {
@Inject
- @V1Impl
- protected EdgeMetadataSerialization serialization;
+ protected EdgeMetadataSerializationV1Impl serialization;
@Override
protected EdgeMetadataSerialization getSerializationImpl() {
- assertTrue(serialization instanceof EdgeMetadataSerializationV1Impl );
return serialization;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7925a4b8/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/EdgeMetaDataSerializationV2Test.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/EdgeMetaDataSerializationV2Test.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/EdgeMetaDataSerializationV2Test.java
index 07396a8..9b34b4e 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/EdgeMetaDataSerializationV2Test.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/EdgeMetaDataSerializationV2Test.java
@@ -22,7 +22,6 @@
package org.apache.usergrid.persistence.graph.serialization;
-import org.apache.usergrid.persistence.core.guice.V2Impl;
import org.junit.runner.RunWith;
import org.apache.usergrid.persistence.core.test.ITRunner;
@@ -32,21 +31,17 @@ import org.apache.usergrid.persistence.graph.serialization.impl.EdgeMetadataSeri
import com.google.inject.Inject;
-import static org.junit.Assert.assertTrue;
-
@RunWith( ITRunner.class )
@UseModules( { TestGraphModule.class } )
public class EdgeMetaDataSerializationV2Test extends EdgeMetadataSerializationTest {
@Inject
- @V2Impl
- protected EdgeMetadataSerialization serialization;
+ protected EdgeMetadataSerializationV2Impl serialization;
@Override
protected EdgeMetadataSerialization getSerializationImpl() {
- assertTrue(serialization instanceof EdgeMetadataSerializationV2Impl );
return serialization;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7925a4b8/stack/corepersistence/map/src/test/java/org/apache/usergrid/persistence/map/guice/TestMapModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/map/src/test/java/org/apache/usergrid/persistence/map/guice/TestMapModule.java b/stack/corepersistence/map/src/test/java/org/apache/usergrid/persistence/map/guice/TestMapModule.java
index 18b79a3..19700c6 100644
--- a/stack/corepersistence/map/src/test/java/org/apache/usergrid/persistence/map/guice/TestMapModule.java
+++ b/stack/corepersistence/map/src/test/java/org/apache/usergrid/persistence/map/guice/TestMapModule.java
@@ -1,21 +1,14 @@
package org.apache.usergrid.persistence.map.guice;
-import org.apache.usergrid.persistence.core.guice.TestModule;
import org.apache.usergrid.persistence.core.guice.CommonModule;
-import org.apache.usergrid.persistence.core.rx.AllEntitiesInSystemObservable;
-import org.apache.usergrid.persistence.core.rx.AllEntitiesInSystemTestObservable;
-import org.apache.usergrid.persistence.core.rx.ApplicationObservable;
-import org.apache.usergrid.persistence.core.rx.ApplicationsTestObservable;
+import org.apache.usergrid.persistence.core.guice.TestModule;
public class TestMapModule extends TestModule {
@Override
protected void configure() {
- bind(AllEntitiesInSystemObservable.class).to(AllEntitiesInSystemTestObservable.class);
- bind(ApplicationObservable.class).to(ApplicationsTestObservable.class);
-
install( new CommonModule());
install( new MapModule() );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7925a4b8/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
index 96202c5..7fccd11 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
@@ -19,26 +19,18 @@
package org.apache.usergrid.persistence.index.guice;
-import org.apache.usergrid.persistence.collection.guice.CollectionModule;
-import org.apache.usergrid.persistence.core.guice.TestModule;
import org.apache.usergrid.persistence.core.guice.CommonModule;
-import org.apache.usergrid.persistence.core.rx.AllEntitiesInSystemObservable;
-import org.apache.usergrid.persistence.core.rx.AllEntitiesInSystemTestObservable;
-import org.apache.usergrid.persistence.core.rx.ApplicationObservable;
-import org.apache.usergrid.persistence.core.rx.ApplicationsTestObservable;
+import org.apache.usergrid.persistence.core.guice.TestModule;
public class TestIndexModule extends TestModule {
@Override
protected void configure() {
- bind(AllEntitiesInSystemObservable.class).to(AllEntitiesInSystemTestObservable.class);
- bind(ApplicationObservable.class).to(ApplicationsTestObservable.class);
install( new CommonModule());
// configure collections and our core astyanax framework
- install( new CollectionModule() );
install( new IndexModule() );
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7925a4b8/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/guice/TestQueueModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/guice/TestQueueModule.java b/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/guice/TestQueueModule.java
index e19b6f0..8390672 100644
--- a/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/guice/TestQueueModule.java
+++ b/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/guice/TestQueueModule.java
@@ -18,21 +18,14 @@
package org.apache.usergrid.persistence.queue.guice;
-import org.apache.usergrid.persistence.core.guice.TestModule;
import org.apache.usergrid.persistence.core.guice.CommonModule;
-import org.apache.usergrid.persistence.core.rx.AllEntitiesInSystemObservable;
-import org.apache.usergrid.persistence.core.rx.AllEntitiesInSystemTestObservable;
-import org.apache.usergrid.persistence.core.rx.ApplicationObservable;
-import org.apache.usergrid.persistence.core.rx.ApplicationsTestObservable;
+import org.apache.usergrid.persistence.core.guice.TestModule;
public class TestQueueModule extends TestModule {
@Override
protected void configure() {
- bind(AllEntitiesInSystemObservable.class).to(AllEntitiesInSystemTestObservable.class);
- bind(ApplicationObservable.class).to(ApplicationsTestObservable.class);
-
install( new CommonModule());
install( new QueueModule() );
}
[25/50] [abbrv] incubator-usergrid git commit: Fixes bug with delete
marking and incorrect return value
Posted by sf...@apache.org.
Fixes bug with delete marking and incorrect return value
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/4ee788d4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/4ee788d4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/4ee788d4
Branch: refs/heads/USERGRID-480
Commit: 4ee788d4db58899ab68778ad97586a26dbe753ef
Parents: cf32557
Author: Todd Nine <tn...@apigee.com>
Authored: Thu Mar 5 03:43:04 2015 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Thu Mar 5 03:43:04 2015 -0700
----------------------------------------------------------------------
.../serialization/impl/MvccEntitySerializationStrategyV2Impl.java | 2 +-
.../serialization/impl/MvccEntitySerializationStrategyV3Impl.java | 2 +-
.../impl/MvccEntitySerializationStrategyImplTest.java | 3 ++-
3 files changed, 4 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4ee788d4/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2Impl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2Impl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2Impl.java
index ed97e80..f679185 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2Impl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2Impl.java
@@ -217,7 +217,7 @@ public class MvccEntitySerializationStrategyV2Impl extends MvccEntitySerializati
// it's been deleted, remove it
if ( STATE_DELETED == state ) {
- return new EntityWrapper( MvccEntity.Status.COMPLETE, Optional.<Entity>absent() );
+ return new EntityWrapper( MvccEntity.Status.DELETED, Optional.<Entity>absent() );
}
Entity storedEntity;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4ee788d4/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3Impl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3Impl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3Impl.java
index ba58ee2..6ad18bb 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3Impl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3Impl.java
@@ -583,7 +583,7 @@ public class MvccEntitySerializationStrategyV3Impl implements MvccEntitySerializ
// it's been deleted, remove it
if ( STATE_DELETED == state ) {
- return new EntityWrapper( MvccEntity.Status.COMPLETE, entityVersion, Optional.<Entity>absent() );
+ return new EntityWrapper( MvccEntity.Status.DELETED, entityVersion, Optional.<Entity>absent() );
}
EntityMap storedEntity;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4ee788d4/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImplTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImplTest.java
index f76230d..3c89b31 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImplTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImplTest.java
@@ -251,6 +251,7 @@ public abstract class MvccEntitySerializationStrategyImplTest {
assertEquals( entityId, returned.getId() );
assertEquals( version, returned.getVersion() );
assertFalse( returned.getEntity().isPresent() );
+ assertEquals( MvccEntity.Status.DELETED, returned.getStatus());
//now delete it
serializationStrategy.delete( context, entityId, version ).execute();
@@ -395,7 +396,7 @@ public abstract class MvccEntitySerializationStrategyImplTest {
final Optional<Entity> empty = Optional.absent();
- MvccEntity clearedV3 = new MvccEntityImpl( id, version3, MvccEntity.Status.COMPLETE, empty );
+ MvccEntity clearedV3 = new MvccEntityImpl( id, version3, MvccEntity.Status.DELETED, empty );
MvccEntity returnedV3 =
serializationStrategy.load( context, Collections.singleton( id ), version3 ).getEntity( id );
[04/50] [abbrv] incubator-usergrid git commit: WIP,
still needs refactored and cleaned up.
Posted by sf...@apache.org.
WIP, still needs refactored and cleaned up.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/d3f8ee61
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/d3f8ee61
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/d3f8ee61
Branch: refs/heads/USERGRID-480
Commit: d3f8ee6184e2a85dc669e836201496748bf35001
Parents: 6652fe5
Author: Todd Nine <tn...@apigee.com>
Authored: Thu Feb 26 18:57:03 2015 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Thu Feb 26 18:57:03 2015 -0700
----------------------------------------------------------------------
.../mvcc/MvccEntityMigrationStrategy.java | 30 --
.../MvccEntitySerializationStrategy.java | 3 +-
.../impl/MvccEntityDataMigrationImpl.java | 179 -----------
...vccEntitySerializationStrategyProxyImpl.java | 18 +-
...cEntitySerializationStrategyProxyV1Impl.java | 62 ----
...cEntitySerializationStrategyProxyV2Impl.java | 43 ---
.../serialization/impl/SerializationModule.java | 22 +-
.../migration/CollectionMigrationPlugin.java | 68 +++++
.../impl/migration/EntityIdScope.java | 49 +++
.../migration/MvccEntityDataMigrationImpl.java | 179 +++++++++++
.../persistence/core/guice/CommonModule.java | 8 +-
.../data/ApplicationDataMigration.java | 2 +-
.../migration/data/CollectionDataMigration.java | 2 +-
.../core/migration/data/DataMigration.java | 94 ------
.../migration/data/DataMigrationManager.java | 15 +-
.../data/DataMigrationManagerImpl.java | 304 ++++++-------------
.../data/MigrationInfoSerialization.java | 30 +-
.../data/MigrationInfoSerializationImpl.java | 84 ++---
.../migration/data/newimpls/DataMigration2.java | 26 +-
.../data/newimpls/MigrationPlugin.java | 12 +-
.../data/newimpls/MigrationRelationship.java | 41 +++
.../data/newimpls/ProgressObserver.java | 50 +++
.../migration/data/newimpls/VersionedData.java | 38 +++
.../migration/data/newimpls/VersionedSet.java | 113 +++++++
.../migration/schema/MigrationStrategy.java | 60 ----
.../core/rx/AllEntitiesInSystemObservable.java | 2 +-
.../core/scope/ApplicationEntityGroup.java | 39 ---
.../persistence/core/scope/EntityIdScope.java | 44 ---
.../core/guice/MaxMigrationVersion.java | 1 -
.../persistence/core/guice/TestModule.java | 3 +-
.../data/DataMigrationManagerImplTest.java | 2 -
.../impl/EdgeDataMigrationImpl.java | 111 +++----
32 files changed, 801 insertions(+), 933 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d3f8ee61/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/MvccEntityMigrationStrategy.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/MvccEntityMigrationStrategy.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/MvccEntityMigrationStrategy.java
deleted file mode 100644
index 6fed25d..0000000
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/MvccEntityMigrationStrategy.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- *
- * * Licensed to the Apache Software Foundation (ASF) under one or more
- * * contributor license agreements. The ASF licenses this file to You
- * * under the Apache License, Version 2.0 (the "License"); you may not
- * * use this file except in compliance with the License.
- * * You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing, software
- * * distributed under the License is distributed on an "AS IS" BASIS,
- * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * * See the License for the specific language governing permissions and
- * * limitations under the License. For additional information regarding
- * * copyright in this work, please see the NOTICE file in the top level
- * * directory of this distribution.
- *
- */
-package org.apache.usergrid.persistence.collection.mvcc;
-
-import org.apache.usergrid.persistence.collection.serialization.MvccEntitySerializationStrategy;
-import org.apache.usergrid.persistence.core.migration.schema.MigrationStrategy;
-
-/**
- * Encapsulates the migration path between versions
- */
-public interface MvccEntityMigrationStrategy extends MigrationStrategy<MvccEntitySerializationStrategy> {
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d3f8ee61/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/MvccEntitySerializationStrategy.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/MvccEntitySerializationStrategy.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/MvccEntitySerializationStrategy.java
index b0b7233..de3cab3 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/MvccEntitySerializationStrategy.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/MvccEntitySerializationStrategy.java
@@ -26,6 +26,7 @@ import java.util.UUID;
import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.collection.EntitySet;
import org.apache.usergrid.persistence.collection.MvccEntity;
+import org.apache.usergrid.persistence.core.migration.data.newimpls.VersionedData;
import org.apache.usergrid.persistence.core.migration.schema.Migration;
import org.apache.usergrid.persistence.model.entity.Id;
@@ -36,7 +37,7 @@ import com.netflix.astyanax.MutationBatch;
/**
* The interface that allows us to serialize an entity to disk
*/
-public interface MvccEntitySerializationStrategy extends Migration {
+public interface MvccEntitySerializationStrategy extends Migration, VersionedData {
/**
* Serialize the entity to the data store with the given collection context
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d3f8ee61/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntityDataMigrationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntityDataMigrationImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntityDataMigrationImpl.java
deleted file mode 100644
index 4dfc64b..0000000
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntityDataMigrationImpl.java
+++ /dev/null
@@ -1,179 +0,0 @@
-/*
- *
- * * Licensed to the Apache Software Foundation (ASF) under one or more
- * * contributor license agreements. The ASF licenses this file to You
- * * under the Apache License, Version 2.0 (the "License"); you may not
- * * use this file except in compliance with the License.
- * * You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing, software
- * * distributed under the License is distributed on an "AS IS" BASIS,
- * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * * See the License for the specific language governing permissions and
- * * limitations under the License. For additional information regarding
- * * copyright in this work, please see the NOTICE file in the top level
- * * directory of this distribution.
- *
- */
-package org.apache.usergrid.persistence.collection.serialization.impl;
-
-
-import java.util.Collections;
-import java.util.List;
-import java.util.UUID;
-import java.util.concurrent.atomic.AtomicLong;
-
-import org.apache.usergrid.persistence.collection.CollectionScope;
-import org.apache.usergrid.persistence.collection.EntitySet;
-import org.apache.usergrid.persistence.collection.MvccEntity;
-import org.apache.usergrid.persistence.collection.mvcc.MvccEntityMigrationStrategy;
-import org.apache.usergrid.persistence.collection.serialization.MvccEntitySerializationStrategy;
-import org.apache.usergrid.persistence.core.migration.data.CollectionDataMigration;
-import org.apache.usergrid.persistence.core.migration.data.DataMigration;
-import org.apache.usergrid.persistence.core.migration.data.DataMigrationException;
-import org.apache.usergrid.persistence.core.migration.schema.MigrationStrategy;
-import org.apache.usergrid.persistence.core.scope.ApplicationEntityGroup;
-import org.apache.usergrid.persistence.core.scope.EntityIdScope;
-import org.apache.usergrid.persistence.model.util.UUIDGenerator;
-
-import com.google.inject.Inject;
-import com.netflix.astyanax.Keyspace;
-import com.netflix.astyanax.MutationBatch;
-import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
-
-import rx.Observable;
-import rx.functions.Action1;
-import rx.functions.Func1;
-import rx.functions.Func2;
-import rx.schedulers.Schedulers;
-
-
-/**
- * Data migration strategy for entities
- */
-public class MvccEntityDataMigrationImpl implements CollectionDataMigration {
-
- private final Keyspace keyspace;
- private final MvccEntityMigrationStrategy entityMigrationStrategy;
-
-
- @Inject
- public MvccEntityDataMigrationImpl( Keyspace keyspace, MvccEntityMigrationStrategy serializationStrategy ) {
-
- this.keyspace = keyspace;
- this.entityMigrationStrategy = serializationStrategy;
- }
-
-
- @Override
- public Observable migrate( final Observable<ApplicationEntityGroup> applicationEntityGroupObservable,
- final DataMigration.ProgressObserver observer ) {
- final AtomicLong atomicLong = new AtomicLong();
-
-
- //capture the time the test starts
- final UUID now = UUIDGenerator.newTimeUUID();
-
- return applicationEntityGroupObservable.flatMap( new Func1<ApplicationEntityGroup, Observable<Long>>() {
- @Override
- public Observable<Long> call( final ApplicationEntityGroup applicationEntityGroup ) {
- final List<EntityIdScope<CollectionScope>> entityIds = applicationEntityGroup.entityIds;
-
- //go through each entity in the system, and load it's entire
- // history
- return Observable.from( entityIds ).subscribeOn( Schedulers.io() )
- .parallel( new Func1<Observable<EntityIdScope<CollectionScope>>, Observable<Long>>() {
-
-
- //process the ids in parallel
- @Override
- public Observable<Long> call(
- final Observable<EntityIdScope<CollectionScope>> entityIdScopeObservable
- ) {
-
- final MutationBatch totalBatch = keyspace.prepareMutationBatch();
-
- return entityIdScopeObservable.doOnNext(
- new Action1<EntityIdScope<CollectionScope>>() {
-
- //load the entity and add it to the toal mutation
- @Override
- public void call( final EntityIdScope<CollectionScope> idScope ) {
-
- //load the entity
- MigrationStrategy
- .MigrationRelationship<MvccEntitySerializationStrategy>
- migration = entityMigrationStrategy.getMigration();
-
-
- CollectionScope currentScope = idScope.getCollectionScope();
- //for each element in the history in the previous
- // version,
- // copy it to the CF in v2
- EntitySet allVersions = migration.from().load( currentScope,
- Collections.singleton( idScope.getId() ), now );
-
- final MvccEntity version =
- allVersions.getEntity( idScope.getId() );
-
- final MutationBatch versionBatch =
- migration.to().write( currentScope, version );
-
- totalBatch.mergeShallow( versionBatch );
- }
- } )
- //every 100 flush the mutation
- .buffer( 100 ).doOnNext(
- new Action1<List<EntityIdScope<CollectionScope>>>() {
- @Override
- public void call(
- final List<EntityIdScope<CollectionScope>> ids ) {
- atomicLong.addAndGet( 100 );
- executeBatch( totalBatch, observer, atomicLong );
- }
- } )
- //count the results
- .reduce( 0l,
- new Func2<Long, List<EntityIdScope<CollectionScope>>, Long>() {
- @Override
- public Long call( final Long aLong,
- final
- List<EntityIdScope<CollectionScope>>
- ids ) {
- return aLong + ids.size();
- }
- } );
- }
- } );
- }
- } );
- }
-
-
- protected void executeBatch( final MutationBatch batch, final DataMigration.ProgressObserver po,
- final AtomicLong count ) {
- try {
- batch.execute();
-
- po.update( getVersion(), "Finished copying " + count + " entities to the new format" );
- }
- catch ( ConnectionException e ) {
- po.failed( getVersion(), "Failed to execute mutation in cassandra" );
- throw new DataMigrationException( "Unable to migrate batches ", e );
- }
- }
-
-
- @Override
- public int getVersion() {
- return entityMigrationStrategy.getVersion();
- }
-
-
- @Override
- public MigrationType getType() {
- return MigrationType.Entities;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d3f8ee61/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyImpl.java
index c18df29..03271a2 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyImpl.java
@@ -35,27 +35,24 @@ import java.util.*;
/**
* Version 4 implementation of entity serialization. This will proxy writes and reads so that during
- * migration data goes to both sources and is read from the old source. After the ugprade completes,
+ * migration data goes to both sources and is read from the old source. After the upgrade completes,
* it will be available from the new source
*/
-public abstract class MvccEntitySerializationStrategyProxyImpl implements MvccEntitySerializationStrategy, MvccEntityMigrationStrategy {
+public class MvccEntitySerializationStrategyProxyImpl implements MvccEntitySerializationStrategy {
protected final Keyspace keyspace;
- protected final MvccEntitySerializationStrategy previous;
- protected final MvccEntitySerializationStrategy current;
private final MigrationInfoSerialization migrationInfoSerialization;
+ private final
@Inject
public MvccEntitySerializationStrategyProxyImpl(final Keyspace keyspace,
- final MvccEntitySerializationStrategy previous,
- final MvccEntitySerializationStrategy current,
+ final Set<MvccEntitySerializationStrategy> allVersions,
final MigrationInfoSerialization migrationInfoSerialization) {
this.keyspace = keyspace;
- this.previous = previous;
- this.current = current;
+
this.migrationInfoSerialization = migrationInfoSerialization;
}
@@ -159,5 +156,10 @@ public abstract class MvccEntitySerializationStrategyProxyImpl implements MvccEn
return Collections.emptyList();
}
+
+ @Override
+ public int getImplementationVersion() {
+ return 0;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d3f8ee61/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV1Impl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV1Impl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV1Impl.java
deleted file mode 100644
index f2c8d33..0000000
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV1Impl.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.persistence.collection.serialization.impl;
-
-
-import org.apache.usergrid.persistence.collection.mvcc.MvccEntityMigrationStrategy;
-import org.apache.usergrid.persistence.collection.serialization.MvccEntitySerializationStrategy;
-import org.apache.usergrid.persistence.core.guice.V1Impl;
-import org.apache.usergrid.persistence.core.guice.V2Impl;
-import org.apache.usergrid.persistence.core.migration.data.DataMigrationManager;
-
-import com.google.inject.Inject;
-import com.google.inject.Singleton;
-import com.netflix.astyanax.Keyspace;
-import org.apache.usergrid.persistence.core.migration.data.MigrationInfoSerialization;
-
-
-/**
- * Version 3 implementation of entity serialization. This will proxy writes and reads so that during
- * migration data goes to both sources and is read from the old source. After the ugprade completes,
- * it will be available from the new source
- */
-@Singleton
-public class MvccEntitySerializationStrategyProxyV1Impl extends MvccEntitySerializationStrategyProxyImpl implements MvccEntityMigrationStrategy {
-
-
- @Inject
- public MvccEntitySerializationStrategyProxyV1Impl(final Keyspace keyspace,
- @V1Impl final MvccEntitySerializationStrategy previous,
- @V2Impl final MvccEntitySerializationStrategy current,
- final MigrationInfoSerialization migrationInfoSerialization) {
- super( keyspace, previous, current,migrationInfoSerialization);
- }
-
- @Override
- public MigrationRelationship<MvccEntitySerializationStrategy> getMigration() {
- return new MigrationRelationship<>(previous,current);
- }
-
- @Override
- public int getVersion() {
- return V2Impl.MIGRATION_VERSION;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d3f8ee61/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV2Impl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV2Impl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV2Impl.java
deleted file mode 100644
index d897ee4..0000000
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV2Impl.java
+++ /dev/null
@@ -1,43 +0,0 @@
-package org.apache.usergrid.persistence.collection.serialization.impl;
-
-import com.google.inject.Inject;
-import com.google.inject.Singleton;
-import com.netflix.astyanax.Keyspace;
-import org.apache.usergrid.persistence.collection.mvcc.MvccEntityMigrationStrategy;
-import org.apache.usergrid.persistence.collection.serialization.MvccEntitySerializationStrategy;
-import org.apache.usergrid.persistence.core.guice.*;
-import org.apache.usergrid.persistence.core.migration.data.DataMigrationManager;
-import org.apache.usergrid.persistence.core.migration.data.MigrationInfoSerialization;
-
-
-/**
- * Version 4 implementation of entity serialization. This will proxy writes and reads so that during
- * migration data goes to both sources and is read from the old source. After the ugprade completes,
- * it will be available from the new source
- */
-@Singleton
-public class MvccEntitySerializationStrategyProxyV2Impl extends MvccEntitySerializationStrategyProxyImpl implements MvccEntityMigrationStrategy {
-
-
- @Inject
- public MvccEntitySerializationStrategyProxyV2Impl( final MigrationInfoSerialization migrationInfoSerialization,
- final Keyspace keyspace,
- @V1ProxyImpl final MvccEntitySerializationStrategy previous,
- @V3Impl final MvccEntitySerializationStrategy current) {
- super(keyspace,previous,current,migrationInfoSerialization);
- }
-
- @Override
- public int getVersion() {
- return V3Impl.MIGRATION_VERSION;
- }
-
-
- @Override
- public MigrationRelationship<MvccEntitySerializationStrategy> getMigration() {
- return new MigrationRelationship<>(this.previous,this.current);
- }
-
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d3f8ee61/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/SerializationModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/SerializationModule.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/SerializationModule.java
index a86d7ec..a9c493f 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/SerializationModule.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/SerializationModule.java
@@ -22,9 +22,15 @@ import org.apache.usergrid.persistence.collection.mvcc.MvccEntityMigrationStrate
import org.apache.usergrid.persistence.collection.mvcc.MvccLogEntrySerializationStrategy;
import org.apache.usergrid.persistence.collection.serialization.MvccEntitySerializationStrategy;
import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
-import org.apache.usergrid.persistence.core.guice.*;
-import org.apache.usergrid.persistence.core.migration.data.CollectionDataMigration;
-import org.apache.usergrid.persistence.core.migration.data.DataMigration;
+import org.apache.usergrid.persistence.collection.serialization.impl.migration.CollectionMigrationPlugin;
+import org.apache.usergrid.persistence.collection.serialization.impl.migration.MvccEntityDataMigrationImpl;
+import org.apache.usergrid.persistence.core.guice.ProxyImpl;
+import org.apache.usergrid.persistence.core.guice.V1Impl;
+import org.apache.usergrid.persistence.core.guice.V1ProxyImpl;
+import org.apache.usergrid.persistence.core.guice.V2Impl;
+import org.apache.usergrid.persistence.core.guice.V3Impl;
+import org.apache.usergrid.persistence.core.migration.data.newimpls.DataMigration2;
+import org.apache.usergrid.persistence.core.migration.data.newimpls.MigrationPlugin;
import org.apache.usergrid.persistence.core.migration.schema.Migration;
import com.google.inject.AbstractModule;
@@ -56,10 +62,16 @@ public class SerializationModule extends AbstractModule {
bind(MvccEntitySerializationStrategy.class ).annotatedWith(ProxyImpl.class)
.to(MvccEntitySerializationStrategyProxyV2Impl.class);
- Multibinder<DataMigration> dataMigrationMultibinder = Multibinder.newSetBinder( binder(), DataMigration.class );
+
+
+
+ Multibinder<DataMigration2> dataMigrationMultibinder = Multibinder.newSetBinder( binder(), DataMigration2.class );
dataMigrationMultibinder.addBinding().to( MvccEntityDataMigrationImpl.class );
- bind( MvccEntityMigrationStrategy.class ).to(MvccEntitySerializationStrategyProxyV2Impl.class);
+
+ Multibinder.newSetBinder( binder(), MigrationPlugin.class).addBinding().to( CollectionMigrationPlugin.class );
+
+
bind( MvccLogEntrySerializationStrategy.class ).to( MvccLogEntrySerializationStrategyImpl.class );
bind( UniqueValueSerializationStrategy.class ).to( UniqueValueSerializationStrategyImpl.class );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d3f8ee61/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/CollectionMigrationPlugin.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/CollectionMigrationPlugin.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/CollectionMigrationPlugin.java
new file mode 100644
index 0000000..5713af7
--- /dev/null
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/CollectionMigrationPlugin.java
@@ -0,0 +1,68 @@
+/*
+ *
+ * *
+ * * Licensed to the Apache Software Foundation (ASF) under one
+ * * or more contributor license agreements. See the NOTICE file
+ * * distributed with this work for additional information
+ * * regarding copyright ownership. The ASF licenses this file
+ * * to you under the Apache License, Version 2.0 (the
+ * * "License"); you may not use this file except in compliance
+ * * with the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing,
+ * * software distributed under the License is distributed on an
+ * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * * KIND, either express or implied. See the License for the
+ * * specific language governing permissions and limitations
+ * * under the License.
+ * *
+ *
+ */
+
+package org.apache.usergrid.persistence.collection.serialization.impl.migration;
+
+
+import java.util.Set;
+
+import org.apache.usergrid.persistence.core.migration.data.newimpls.DataMigration2;
+import org.apache.usergrid.persistence.core.migration.data.newimpls.MigrationDataProvider;
+import org.apache.usergrid.persistence.core.migration.data.newimpls.MigrationPlugin;
+import org.apache.usergrid.persistence.core.migration.data.newimpls.ProgressObserver;
+
+
+/**
+ * Migration plugin for the collection module
+ */
+public class CollectionMigrationPlugin implements MigrationPlugin {
+
+
+ private final DataMigration2<EntityIdScope> entityDataMigration;
+ private final MigrationDataProvider<EntityIdScope> entityIdScopeDataMigrationProvider;
+
+
+ public CollectionMigrationPlugin( final DataMigration2<EntityIdScope> entityDataMigration,
+ final MigrationDataProvider<EntityIdScope> entityIdScopeDataMigrationProvider ) {
+ this.entityDataMigration = entityDataMigration;
+ this.entityIdScopeDataMigrationProvider = entityIdScopeDataMigrationProvider;
+ }
+
+
+ @Override
+ public String getName() {
+ return "collections-entity-data";
+ }
+
+
+ @Override
+ public void run( final ProgressObserver observer ) {
+ entityDataMigration.migrate( entityIdScopeDataMigrationProvider, observer );
+ }
+
+
+ @Override
+ public int getMaxVersion() {
+ return 0;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d3f8ee61/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/EntityIdScope.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/EntityIdScope.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/EntityIdScope.java
new file mode 100644
index 0000000..0e70e75
--- /dev/null
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/EntityIdScope.java
@@ -0,0 +1,49 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. The ASF licenses this file to You
+ * * under the Apache License, Version 2.0 (the "License"); you may not
+ * * use this file except in compliance with the License.
+ * * You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License. For additional information regarding
+ * * copyright in this work, please see the NOTICE file in the top level
+ * * directory of this distribution.
+ *
+ */
+package org.apache.usergrid.persistence.collection.serialization.impl.migration;
+
+import java.util.Collection;
+
+import org.apache.usergrid.persistence.collection.CollectionScope;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+/**
+ * Tuple containing collectionscope and entityid
+ */
+public class EntityIdScope{
+ private final Id id;
+ private final CollectionScope collectionScope;
+
+ public EntityIdScope(CollectionScope collectionScope, Id id){
+ this.id = id;
+ this.collectionScope = collectionScope;
+ }
+
+
+ public Id getId() {
+ return id;
+ }
+
+
+ public CollectionScope getCollectionScope() {
+ return collectionScope;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d3f8ee61/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/MvccEntityDataMigrationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/MvccEntityDataMigrationImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/MvccEntityDataMigrationImpl.java
new file mode 100644
index 0000000..73efcba
--- /dev/null
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/MvccEntityDataMigrationImpl.java
@@ -0,0 +1,179 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. The ASF licenses this file to You
+ * * under the Apache License, Version 2.0 (the "License"); you may not
+ * * use this file except in compliance with the License.
+ * * You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License. For additional information regarding
+ * * copyright in this work, please see the NOTICE file in the top level
+ * * directory of this distribution.
+ *
+ */
+package org.apache.usergrid.persistence.collection.serialization.impl.migration;
+
+
+import java.util.Collections;
+import java.util.List;
+import java.util.UUID;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.usergrid.persistence.collection.CollectionScope;
+import org.apache.usergrid.persistence.collection.EntitySet;
+import org.apache.usergrid.persistence.collection.MvccEntity;
+import org.apache.usergrid.persistence.collection.mvcc.MvccEntityMigrationStrategy;
+import org.apache.usergrid.persistence.collection.serialization.MvccEntitySerializationStrategy;
+import org.apache.usergrid.persistence.core.migration.data.DataMigrationException;
+import org.apache.usergrid.persistence.core.migration.data.newimpls.DataMigration2;
+import org.apache.usergrid.persistence.core.migration.data.newimpls.MigrationDataProvider;
+import org.apache.usergrid.persistence.core.migration.data.newimpls.ProgressObserver;
+import org.apache.usergrid.persistence.core.migration.schema.MigrationStrategy;
+import org.apache.usergrid.persistence.model.util.UUIDGenerator;
+
+import com.google.inject.Inject;
+import com.netflix.astyanax.Keyspace;
+import com.netflix.astyanax.MutationBatch;
+import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
+
+import rx.Observable;
+import rx.functions.Action1;
+import rx.functions.Func1;
+import rx.functions.Func2;
+import rx.schedulers.Schedulers;
+
+
+/**
+ * Data migration strategy for entities
+ */
+public class MvccEntityDataMigrationImpl implements DataMigration2<EntityIdScope> {
+
+ private final Keyspace keyspace;
+ private final MvccEntityMigrationStrategy entityMigrationStrategy;
+
+
+ @Inject
+ public MvccEntityDataMigrationImpl( Keyspace keyspace, MvccEntityMigrationStrategy serializationStrategy ) {
+
+ this.keyspace = keyspace;
+ this.entityMigrationStrategy = serializationStrategy;
+ }
+
+
+ @Override
+ public int getVersion() {
+ return entityMigrationStrategy.getVersion();
+ }
+
+
+ @Override
+ public void migrate( final MigrationDataProvider<EntityIdScope> migrationDataProvider, final ProgressObserver observer ) {
+
+ final AtomicLong atomicLong = new AtomicLong();
+
+
+ //capture the time the test starts
+
+ final UUID startTime = UUIDGenerator.newTimeUUID();
+
+ final long migrated = migrationDataProvider.getData().subscribeOn( Schedulers.io() )
+ .parallel( new Func1<Observable<EntityIdScope>, Observable<Long>>() {
+
+
+ //process the ids in parallel
+ @Override
+ public Observable<Long> call(
+ final Observable<EntityIdScope>
+ entityIdScopeObservable ) {
+
+ final MutationBatch totalBatch =
+ keyspace.prepareMutationBatch();
+
+ return entityIdScopeObservable.doOnNext(
+ new Action1<EntityIdScope>() {
+
+ //load the entity and add it to the toal mutation
+ @Override
+ public void call( final EntityIdScope idScope ) {
+
+ //load the entity
+ MigrationStrategy
+ .MigrationRelationship<MvccEntitySerializationStrategy>
+ migration = entityMigrationStrategy
+ .getMigration();
+
+
+ CollectionScope currentScope =
+ idScope.getCollectionScope();
+
+ //for each element in the history in the
+ // previous
+ // version,
+ // copy it to the CF in v2
+
+
+ EntitySet allVersions = migration.from()
+ .load( currentScope,
+ Collections
+ .singleton(
+ idScope.getId() ),
+ startTime );
+
+ final MvccEntity version = allVersions
+ .getEntity( idScope.getId() );
+
+ final MutationBatch versionBatch =
+ migration.to().write( currentScope,
+ version );
+
+ totalBatch.mergeShallow( versionBatch );
+ }
+ } )
+ //every 100 flush the mutation
+ .buffer( 100 ).doOnNext(
+ new Action1<List<EntityIdScope>>() {
+ @Override
+ public void call(
+ final List<EntityIdScope> ids ) {
+ atomicLong.addAndGet( 100 );
+ executeBatch( totalBatch, observer,
+ atomicLong );
+ }
+ } )
+ //count the results
+ .reduce( 0l,
+ new Func2<Long, List<EntityIdScope>, Long>
+ () {
+ @Override
+ public Long call( final Long aLong,
+ final List<EntityIdScope> ids ) {
+ return aLong + ids.size();
+ }
+ } );
+ }
+ } ).toBlocking().last();
+
+ //now we update the progress observer
+
+ observer.update( getVersion(), "Finished for this step. Migrated " + migrated + "entities total. ");
+ }
+
+
+ protected void executeBatch( final MutationBatch batch, final ProgressObserver po, final AtomicLong count ) {
+ try {
+ batch.execute();
+
+ po.update( getVersion(), "Finished copying " + count + " entities to the new format" );
+ }
+ catch ( ConnectionException e ) {
+ po.failed( getVersion(), "Failed to execute mutation in cassandra" );
+ throw new DataMigrationException( "Unable to migrate batches ", e );
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d3f8ee61/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/guice/CommonModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/guice/CommonModule.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/guice/CommonModule.java
index 657d965..61b6b04 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/guice/CommonModule.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/guice/CommonModule.java
@@ -28,6 +28,7 @@ import org.apache.usergrid.persistence.core.astyanax.CassandraConfigImpl;
import org.apache.usergrid.persistence.core.astyanax.CassandraFig;
import org.apache.usergrid.persistence.core.consistency.TimeService;
import org.apache.usergrid.persistence.core.consistency.TimeServiceImpl;
+import org.apache.usergrid.persistence.core.migration.data.newimpls.MigrationPlugin;
import org.apache.usergrid.persistence.core.migration.schema.Migration;
import org.apache.usergrid.persistence.core.migration.schema.MigrationManager;
import org.apache.usergrid.persistence.core.migration.schema.MigrationManagerFig;
@@ -77,11 +78,8 @@ public class CommonModule extends AbstractModule {
//do multibindings for migrations
- Multibinder<DataMigration> applicationDataMigrationMultibinder = Multibinder.newSetBinder( binder(), DataMigration.class );
-// dataMigrationMultibinder.addBinding();
-// dataMigrationManagerMultibinder.addBinding().to( DataMigrationManagerImpl.class );
-// migrationBinding.addBinding().to( Key.get( MigrationInfoSerialization.class ) );
-
+ //create the empty multibinder so other plugins can use it
+ Multibinder.newSetBinder( binder(), MigrationPlugin.class);
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d3f8ee61/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/ApplicationDataMigration.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/ApplicationDataMigration.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/ApplicationDataMigration.java
index 6dca092..bbff0b9 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/ApplicationDataMigration.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/ApplicationDataMigration.java
@@ -21,7 +21,7 @@ package org.apache.usergrid.persistence.core.migration.data;
import org.apache.usergrid.persistence.core.scope.ApplicationEntityGroup;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import rx.Observable;
+
/**
* Migrate applications
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d3f8ee61/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/CollectionDataMigration.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/CollectionDataMigration.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/CollectionDataMigration.java
index e29571c..df1af72 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/CollectionDataMigration.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/CollectionDataMigration.java
@@ -20,7 +20,7 @@
package org.apache.usergrid.persistence.core.migration.data;
import org.apache.usergrid.persistence.core.scope.ApplicationEntityGroup;
-import rx.Observable;
+
/**
* Migrate Collections
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d3f8ee61/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/DataMigration.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/DataMigration.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/DataMigration.java
deleted file mode 100644
index e1e3092..0000000
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/DataMigration.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.usergrid.persistence.core.migration.data;
-
-
-import org.apache.usergrid.persistence.core.scope.ApplicationEntityGroup;
-import rx.Observable;
-
-/**
- * An interface for updating data. Has 2 basic functions. First it will perform the migration and update the status
- * object.
- *
- * Second it will only migrate a single version. For instance, it will migrate from 0->1, or from 1->2. All migrations
- * must follow the following basic rules.
- *
- * <ol>
- * <li>They must not modify the structure of an existing column family. If the data format changes, a new
- * implementation and column family must be created. A proxy must be defined to do dual writes/single reads. </li>
- * <li>The migration must update the progress observer. This information should be made available cluster wide.</li>
- * <li>In the event a migration fails with an error, we should be able to roll back and remove new column families. We
- * can then fix the bug, and deploy again. Hence the need for the proxy, dual writes, and an immutable CF
- * format</li>
- * </ol>
- */
-
-
-public interface DataMigration <T> {
-
-
- /**
- * Migrate the data to the specified version
- * @param observer
- * @Return an observable containing the count of the number of elements migrated
- * @throws Throwable
- */
- public Observable<Long> migrate(final Observable<T> applicationEntityGroup,final ProgressObserver observer) throws Throwable;
-
- /**
- * Get the version of this migration. It must be unique.
- * @return
- */
- public int getVersion();
-
- public MigrationType getType();
-
- public interface ProgressObserver{
- /**
- * Mark the migration as failed
- * @param migrationVersion The migration version running during the failure
- * @param reason The reason to save
- */
- public void failed(final int migrationVersion, final String reason);
-
- /**
- * Mark the migration as failed with a stack trace
- * @param migrationVersion The migration version running during the failure
- * @param reason The error description to save
- * @param throwable The error that happened
- */
- public void failed(final int migrationVersion, final String reason, final Throwable throwable);
-
-
- /**
- * Update the status of the migration with the message
- *
- * @param message The message to save for the status
- */
- public void update(final int migrationVersion, final String message);
- }
-
- public enum MigrationType{
- Entities,
- Applications,
- System,
- Collections
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d3f8ee61/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManager.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManager.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManager.java
index e47e264..1ca8d90 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManager.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManager.java
@@ -19,6 +19,9 @@
package org.apache.usergrid.persistence.core.migration.data;
+import java.util.List;
+import java.util.Set;
+
import org.apache.usergrid.persistence.core.migration.schema.MigrationException;
@@ -43,13 +46,13 @@ public interface DataMigrationManager {
* Get the current version of the schema
* @return
*/
- public int getCurrentVersion();
+ public int getCurrentVersion(final String pluginName);
/**
* Reset the system version to the version specified
* @param version
*/
- public void resetToVersion(final int version);
+ public void resetToVersion(final String pluginName, final int version);
/**
* Invalidate the cache for versions
@@ -60,5 +63,11 @@ public interface DataMigrationManager {
/**
* Return that last status of the migration
*/
- public String getLastStatus();
+ public String getLastStatus(final String pluginName);
+
+ /**
+ * Return a list of all plugin names
+ * @return
+ */
+ public Set<String> getPluginNames();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d3f8ee61/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManagerImpl.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManagerImpl.java
index 380bedc..ddb0a61 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManagerImpl.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManagerImpl.java
@@ -21,24 +21,24 @@ package org.apache.usergrid.persistence.core.migration.data;
import java.io.PrintWriter;
import java.io.StringWriter;
-import java.util.*;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
-import org.apache.usergrid.persistence.core.rx.AllEntitiesInSystemObservable;
-import org.apache.usergrid.persistence.core.rx.ApplicationObservable;
-import org.apache.usergrid.persistence.core.scope.ApplicationEntityGroup;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.usergrid.persistence.core.migration.data.newimpls.MigrationPlugin;
+import org.apache.usergrid.persistence.core.migration.data.newimpls.ProgressObserver;
import org.apache.usergrid.persistence.core.migration.schema.MigrationException;
import com.google.common.base.Preconditions;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
import com.google.inject.Inject;
import com.google.inject.Singleton;
-import rx.Observable;
-import rx.functions.Action1;
-import rx.functions.Func1;
-import rx.schedulers.Schedulers;
@Singleton
@@ -46,256 +46,122 @@ public class DataMigrationManagerImpl implements DataMigrationManager {
private static final Logger LOG = LoggerFactory.getLogger( DataMigrationManagerImpl.class );
- private final TreeMap<Integer, DataMigration> migrationTreeMap = new TreeMap<>();
+ private final Map<String, MigrationPlugin> migrationPlugins = new HashMap<>();
private final MigrationInfoSerialization migrationInfoSerialization;
- private final AllEntitiesInSystemObservable allEntitiesInSystemObservable;
- private final ApplicationObservable applicationObservable;
- private final Set<DataMigration> dataMigrations;
-
-
- @Inject
- public DataMigrationManagerImpl( final MigrationInfoSerialization migrationInfoSerialization,
- final Set<DataMigration> dataMigrations,
- final AllEntitiesInSystemObservable allEntitiesInSystemObservable,
- final ApplicationObservable applicationObservable
- ) {
-
- Preconditions.checkNotNull( migrationInfoSerialization,
- "migrationInfoSerialization must not be null" );
- Preconditions.checkNotNull( dataMigrations, "migrations must not be null" );
- Preconditions.checkNotNull( allEntitiesInSystemObservable, "allentitiesobservable must not be null" );
- Preconditions.checkNotNull( applicationObservable, "applicationObservable must not be null" );
- this.dataMigrations = dataMigrations;
- this.migrationInfoSerialization = migrationInfoSerialization;
- this.allEntitiesInSystemObservable = allEntitiesInSystemObservable;
- this.applicationObservable = applicationObservable;
-
- }
-
-
- @Override
- public void migrate() throws MigrationException {
-
- if (!populateTreeMap())
- return;
-
- int currentVersion = migrationInfoSerialization.getCurrentVersion();
-
- if(currentVersion <= 0){
- resetToHighestVersion();
- //you have no work to do, just use the latest
- return;
- }
-
- LOG.info( "Saved schema version is {}, max migration version is {}", currentVersion,
- migrationTreeMap.lastKey() );
-
- //we have our migrations to run, execute them
- final Collection< DataMigration> migrationsToRun = migrationTreeMap.tailMap( currentVersion, false ).values();
-
-
-
- final CassandraProgressObserver observer = new CassandraProgressObserver();
-
- final Observable<ApplicationScope> appScopeObservable = applicationObservable.getAllApplicationScopes();
-
- final Observable<ApplicationEntityGroup> entitiesObservable = appScopeObservable.flatMap(new Func1<ApplicationScope, Observable<ApplicationEntityGroup>>() {
- @Override
- public Observable<ApplicationEntityGroup> call(ApplicationScope applicationScope) {
- return allEntitiesInSystemObservable.getAllEntitiesInSystem(appScopeObservable, 1000);
- }
- });
-
- final Observable<ApplicationDataMigration> applicationMigrationsRx = Observable.from(migrationsToRun)
- .filter(new Func1<DataMigration, Boolean>() {
- @Override
- public Boolean call(DataMigration dataMigration) {
- return dataMigration instanceof ApplicationDataMigration;
- }
- }).map(new Func1<DataMigration, ApplicationDataMigration>() {
- @Override
- public ApplicationDataMigration call(DataMigration dataMigration) {
- return (ApplicationDataMigration)dataMigration;
- }
- });
-
- final Observable<CollectionDataMigration> collectionMigrationsRx = Observable.from(migrationsToRun)
- .filter(new Func1<DataMigration, Boolean>() {
- @Override
- public Boolean call(DataMigration dataMigration) {
- return dataMigration instanceof CollectionDataMigration;
- }
- }).map(new Func1<DataMigration, CollectionDataMigration>() {
+ /**
+ * Cache to cache versions temporarily
+ */
+ private final LoadingCache<String, Integer> versionCache = CacheBuilder.newBuilder()
+ //cache the local value for 1 minute
+ .expireAfterWrite( 1, TimeUnit.MINUTES ).build( new CacheLoader<String, Integer>() {
@Override
- public CollectionDataMigration call(DataMigration dataMigration) {
- return (CollectionDataMigration) dataMigration;
+ public Integer load( final String key ) throws Exception {
+ return migrationInfoSerialization.getVersion( key );
}
- });
-
- Observable applications = applicationMigrationsRx
- .doOnNext(new Action1<ApplicationDataMigration>() {
- @Override
- public void call(ApplicationDataMigration dataMigration) {
-
- migrationInfoSerialization.setStatusCode(StatusCode.RUNNING.status);
-
- final int migrationVersion = dataMigration.getVersion();
-
- LOG.info("Running migration version {}", migrationVersion);
-
- observer.update(migrationVersion, "Starting migration");
+ } );
- //perform this migration, if it fails, short circuit
- try {
- dataMigration.migrate(appScopeObservable, observer).toBlocking().lastOrDefault(null);
- } catch (Throwable throwable) {
- observer.failed(migrationVersion, "Exception thrown during migration", throwable);
- LOG.error("Unable to migrate to version {}.", migrationVersion, throwable);
- throw new RuntimeException(throwable);
- }
- //we had an unhandled exception or the migration failed, short circuit
- if (observer.failed) {
- return;
- }
-
- //set the version
- migrationInfoSerialization.setVersion(migrationVersion);
-
- //update the observer for progress so other nodes can see it
- observer.update(migrationVersion, "Completed successfully");
-
- }
- });
-
-
- Observable entities =collectionMigrationsRx
- .doOnNext(new Action1<CollectionDataMigration>() {
- @Override
- public void call(CollectionDataMigration dataMigration) {
- migrationInfoSerialization.setStatusCode(StatusCode.RUNNING.status);
-
- final int migrationVersion = dataMigration.getVersion();
+ @Inject
+ public DataMigrationManagerImpl( final Set<MigrationPlugin> plugins,
+ final MigrationInfoSerialization migrationInfoSerialization ) {
- LOG.info("Running migration version {}", migrationVersion);
+ Preconditions.checkNotNull( plugins, "plugins must not be null" );
+ Preconditions.checkNotNull( migrationInfoSerialization, "migrationInfoSerialization must not be null" );
- observer.update(migrationVersion, "Starting migration");
+ this.migrationInfoSerialization = migrationInfoSerialization;
- //perform this migration, if it fails, short circuit
- try {
- dataMigration.migrate(entitiesObservable, observer).toBlocking().lastOrDefault(null);
- } catch (Throwable throwable) {
- observer.failed(migrationVersion, "Exception thrown during migration", throwable);
- LOG.error("Unable to migrate to version {}.", migrationVersion, throwable);
- throw new RuntimeException(throwable);
- }
- //we had an unhandled exception or the migration failed, short circuit
- if (observer.failed) {
- return;
- }
+ for ( MigrationPlugin plugin : plugins ) {
+ final String name = plugin.getName();
- //set the version
- migrationInfoSerialization.setVersion(migrationVersion);
- //update the observer for progress so other nodes can see it
- observer.update(migrationVersion, "Completed successfully");
- }
- });
+ final MigrationPlugin existing = migrationPlugins.get( name );
+ if ( existing != null ) {
+ throw new IllegalArgumentException( "Duplicate plugin name detected. A plugin with name " + name
+ + " is already implemented by class '" + existing.getClass().getName() + "'. Class '" + plugin
+ .getClass().getName() + "' is also trying to implement this name." );
+ }
- try {
- Observable
- .merge(applications, entities)
- .subscribeOn(Schedulers.io())
- .toBlocking().lastOrDefault(null);
- migrationInfoSerialization.setStatusCode( StatusCode.COMPLETE.status );
- } catch (Exception e){
- LOG.error("Migration Failed",e);
+ migrationPlugins.put( name, plugin );
}
-
}
+ @Override
+ public void migrate() throws MigrationException {
- private boolean populateTreeMap() {
- if ( migrationTreeMap.isEmpty() ) {
- for ( DataMigration migration : dataMigrations ) {
-
- Preconditions.checkNotNull(migration,
- "A migration instance in the set of migrations was null. This is not allowed");
-
- final int version = migration.getVersion();
-
- final DataMigration existing = migrationTreeMap.get( version );
-
- if ( existing != null ) {
+ /**
+ * Invoke each plugin to attempt a migration
+ */
+ for(final MigrationPlugin plugin: migrationPlugins.values()){
+ final ProgressObserver observer = new CassandraProgressObserver(plugin.getName());
- final Class<? extends DataMigration> existingClass = existing.getClass();
+ plugin.run( observer );
+ }
- final Class<? extends DataMigration> currentClass = migration.getClass();
+ }
- throw new DataMigrationException( String.format(
- "Data migrations must be unique. Both classes %s and %s have version %d",
- existingClass, currentClass, version ) );
- }
- migrationTreeMap.put( version, migration );
- }
+ @Override
+ public boolean isRunning() {
+ for(final String pluginName :getPluginNames()){
+ if( migrationInfoSerialization.getStatusCode(pluginName) == StatusCode.RUNNING.status){
+ return true;
+ }
}
- if(migrationTreeMap.isEmpty()) {
- LOG.warn("No migrations found to run, exiting");
- return false;
- }
- return true;
+
+ return false;
}
@Override
- public boolean isRunning() {
- return migrationInfoSerialization.getStatusCode() == StatusCode.RUNNING.status;
+ public void invalidate() {
+ versionCache.invalidateAll();
}
@Override
- public void invalidate() {
- migrationInfoSerialization.invalidate();
+ public int getCurrentVersion( final String pluginName ) {
+ Preconditions.checkNotNull( pluginName, "pluginName cannot be null" );
+ return migrationInfoSerialization.getVersion( pluginName );
}
@Override
- public int getCurrentVersion() {
- return migrationInfoSerialization.getCurrentVersion();
- }
+ public void resetToVersion( final String pluginName, final int version ) {
+ Preconditions.checkNotNull( pluginName, "pluginName cannot be null" );
+ final MigrationPlugin plugin = migrationPlugins.get( pluginName );
+ Preconditions.checkArgument( plugin != null, "Plugin " + pluginName + " could not be found" );
- @Override
- public void resetToVersion( final int version ) {
- final int highestAllowed = migrationTreeMap.lastKey();
+ final int highestAllowed = plugin.getMaxVersion();
Preconditions.checkArgument( version <= highestAllowed,
- "You cannot set a version higher than the max of " + highestAllowed);
+ "You cannot set a version higher than the max of " + highestAllowed );
Preconditions.checkArgument( version >= 0, "You must specify a version of 0 or greater" );
- migrationInfoSerialization.setVersion( version );
+ migrationInfoSerialization.setVersion( pluginName, version );
}
- private int resetToHighestVersion( ) {
- final int highestAllowed = migrationTreeMap.lastKey();
- resetToVersion(highestAllowed);
- return highestAllowed;
+
+ @Override
+ public String getLastStatus( final String pluginName ) {
+ Preconditions.checkNotNull( pluginName, "pluginName cannot be null" );
+ return migrationInfoSerialization.getStatusMessage( pluginName );
}
+
@Override
- public String getLastStatus() {
- return migrationInfoSerialization.getStatusMessage();
+ public Set<String> getPluginNames() {
+ return migrationPlugins.keySet();
}
@@ -314,16 +180,20 @@ public class DataMigrationManagerImpl implements DataMigrationManager {
}
- private final class CassandraProgressObserver implements DataMigration.ProgressObserver {
+ private final class CassandraProgressObserver implements ProgressObserver {
+
+ private final String pluginName;
private boolean failed = false;
+ private CassandraProgressObserver( final String pluginName ) {this.pluginName = pluginName;}
+
+
@Override
public void failed( final int migrationVersion, final String reason ) {
- final String storedMessage = String.format(
- "Failed to migrate, reason is appended. Error '%s'", reason );
+ final String storedMessage = String.format( "Failed to migrate, reason is appended. Error '%s'", reason );
update( migrationVersion, storedMessage );
@@ -332,7 +202,7 @@ public class DataMigrationManagerImpl implements DataMigrationManager {
failed = true;
- migrationInfoSerialization.setStatusCode( StatusCode.ERROR.status );
+ migrationInfoSerialization.setStatusCode( pluginName, StatusCode.ERROR.status );
}
@@ -342,30 +212,28 @@ public class DataMigrationManagerImpl implements DataMigrationManager {
throwable.printStackTrace( new PrintWriter( stackTrace ) );
- final String storedMessage = String.format(
- "Failed to migrate, reason is appended. Error '%s' %s", reason, stackTrace.toString() );
+ final String storedMessage = String.format( "Failed to migrate, reason is appended. Error '%s' %s", reason,
+ stackTrace.toString() );
update( migrationVersion, storedMessage );
- LOG.error( "Unable to migrate version {} due to reason {}.",
- migrationVersion, reason, throwable );
+ LOG.error( "Unable to migrate version {} due to reason {}.", migrationVersion, reason, throwable );
failed = true;
- migrationInfoSerialization.setStatusCode( StatusCode.ERROR.status );
+ migrationInfoSerialization.setStatusCode( pluginName, StatusCode.ERROR.status );
}
@Override
public void update( final int migrationVersion, final String message ) {
- final String formattedOutput = String.format(
- "Migration version %d. %s", migrationVersion, message );
+ final String formattedOutput = String.format( "Migration version %d. %s", migrationVersion, message );
//Print this to the info log
LOG.info( formattedOutput );
- migrationInfoSerialization.setStatusMessage( formattedOutput );
+ migrationInfoSerialization.setStatusMessage( pluginName, formattedOutput );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d3f8ee61/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/MigrationInfoSerialization.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/MigrationInfoSerialization.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/MigrationInfoSerialization.java
index 3caa6aa..a972f8c 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/MigrationInfoSerialization.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/MigrationInfoSerialization.java
@@ -26,42 +26,48 @@ public interface MigrationInfoSerialization extends Migration {
/**
* Save the message to the cluster
+ * @param pluginName the name of hte plugin
* @param message
*/
- public void setStatusMessage( final String message );
+ public void setStatusMessage(final String pluginName, final String message );
/**
- * Get the last status
+ * Get the last status for the plugin
* @return
*/
- public String getStatusMessage();
+ public String getStatusMessage(final String pluginName);
/**
- * Save the version
+ * Save the version for the plugin
* @param version
*/
- public void setVersion(final int version);
+ public void setVersion(final String pluginName, final int version);
/**
* Return the version
* @return
*/
- public int getVersion();
+ public int getVersion(final String pluginName);
/**
* Set the status and save them
+ * @param pluginName The name of the plugin
* @param status
* @return
*/
- public void setStatusCode( final int status );
+ public void setStatusCode(final String pluginName, final int status );
/**
- *
+ * Get the status code for the plugin
* @return The integer that's saved
*/
- public int getStatusCode();
+ public int getStatusCode(final String pluginName);
- public int getCurrentVersion();
-
- public void invalidate();
+ /**
+ * This is deprecated, and will be used to migrate from the old version information to the new format.
+ * Should return -1 if not set
+ * @return
+ */
+ @Deprecated
+ public int getSystemVersion();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d3f8ee61/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/MigrationInfoSerializationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/MigrationInfoSerializationImpl.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/MigrationInfoSerializationImpl.java
index aeee0bb..f976568 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/MigrationInfoSerializationImpl.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/MigrationInfoSerializationImpl.java
@@ -50,18 +50,6 @@ import com.netflix.astyanax.serializers.StringSerializer;
public class MigrationInfoSerializationImpl implements MigrationInfoSerialization {
/**
- * Cache to cache versions temporarily
- */
- private final LoadingCache<String, Integer> versionCache = CacheBuilder.newBuilder()
- //cache the local value for 1 minute
- .expireAfterWrite(1, TimeUnit.MINUTES).build( new CacheLoader<String, Integer>() {
- @Override
- public Integer load( final String key ) throws Exception {
- return getVersion();
- }
- } );
-
- /**
* Just a hard coded scope since we need it
*/
private static final Id STATIC_ID =
@@ -78,7 +66,10 @@ public class MigrationInfoSerializationImpl implements MigrationInfoSerializatio
new MultiTennantColumnFamily<>( "Data_Migration_Info", ROW_KEY_SER, STRING_SERIALIZER );
- private static final ScopedRowKey<String> ROW_KEY = ScopedRowKey.fromKey( STATIC_ID, "" );
+ /**
+ * The row key we previously used to store versions. This is required to migrate versions in each module to the correct version
+ */
+ private static final ScopedRowKey<String> LEGACY_ROW_KEY = ScopedRowKey.fromKey( STATIC_ID, "" );
private static final String COL_STATUS_MESSAGE = "statusMessage";
@@ -96,10 +87,12 @@ public class MigrationInfoSerializationImpl implements MigrationInfoSerializatio
@Override
- public void setStatusMessage( final String message ) {
+ public void setStatusMessage(final String pluginName, final String message ) {
+
+ final ScopedRowKey<String> rowKey = ScopedRowKey.fromKey( STATIC_ID, pluginName);
try {
- keyspace.prepareColumnMutation( CF_MIGRATION_INFO, ROW_KEY, COL_STATUS_MESSAGE ).putValue( message, null )
+ keyspace.prepareColumnMutation( CF_MIGRATION_INFO, rowKey, COL_STATUS_MESSAGE ).putValue( message, null )
.execute();
}
catch ( ConnectionException e ) {
@@ -109,9 +102,12 @@ public class MigrationInfoSerializationImpl implements MigrationInfoSerializatio
@Override
- public String getStatusMessage() {
+ public String getStatusMessage(final String pluginName) {
+
+ final ScopedRowKey<String> rowKey = ScopedRowKey.fromKey( STATIC_ID, pluginName);
+
try {
- return keyspace.prepareQuery( CF_MIGRATION_INFO ).getKey( ROW_KEY ).getColumn( COL_STATUS_MESSAGE )
+ return keyspace.prepareQuery( CF_MIGRATION_INFO ).getKey( rowKey ).getColumn( COL_STATUS_MESSAGE )
.execute().getResult().getStringValue();
}
//swallow, it doesn't exist
@@ -125,28 +121,32 @@ public class MigrationInfoSerializationImpl implements MigrationInfoSerializatio
@Override
- public void setVersion( final int version ) {
+ public void setVersion(final String pluginName, final int version ) {
+
+ final ScopedRowKey<String> rowKey = ScopedRowKey.fromKey( STATIC_ID, pluginName);
+
try {
- keyspace.prepareColumnMutation( CF_MIGRATION_INFO, ROW_KEY, COLUMN_VERSION ).putValue( version, null )
+ keyspace.prepareColumnMutation( CF_MIGRATION_INFO, rowKey, COLUMN_VERSION ).putValue( version, null )
.execute();
}
catch ( ConnectionException e ) {
throw new DataMigrationException( "Unable to save status", e );
}
-
- versionCache.invalidateAll();
}
@Override
- public int getVersion() {
+ public int getVersion(final String pluginName) {
+
+ final ScopedRowKey<String> rowKey = ScopedRowKey.fromKey( STATIC_ID, pluginName);
+
try {
- return keyspace.prepareQuery( CF_MIGRATION_INFO ).getKey( ROW_KEY ).getColumn( COLUMN_VERSION ).execute()
+ return keyspace.prepareQuery( CF_MIGRATION_INFO ).getKey( rowKey ).getColumn( COLUMN_VERSION ).execute()
.getResult().getIntegerValue();
}
//swallow, it doesn't exist
catch ( NotFoundException nfe ) {
- return 0;
+ return -1;
}
catch ( ConnectionException e ) {
throw new DataMigrationException( "Unable to retrieve status", e );
@@ -155,9 +155,12 @@ public class MigrationInfoSerializationImpl implements MigrationInfoSerializatio
@Override
- public void setStatusCode( final int status ) {
+ public void setStatusCode(final String pluginName, final int status ) {
+
+ final ScopedRowKey<String> rowKey = ScopedRowKey.fromKey( STATIC_ID, pluginName);
+
try {
- keyspace.prepareColumnMutation( CF_MIGRATION_INFO, ROW_KEY, COLUMN_STATUS_CODE ).putValue( status, null )
+ keyspace.prepareColumnMutation( CF_MIGRATION_INFO, rowKey, COLUMN_STATUS_CODE ).putValue( status, null )
.execute();
}
catch ( ConnectionException e ) {
@@ -167,32 +170,37 @@ public class MigrationInfoSerializationImpl implements MigrationInfoSerializatio
@Override
- public int getStatusCode() {
+ public int getStatusCode(final String pluginName) {
+
+ final ScopedRowKey<String> rowKey = ScopedRowKey.fromKey( STATIC_ID, pluginName);
+
try {
- return keyspace.prepareQuery( CF_MIGRATION_INFO ).getKey( ROW_KEY ).getColumn( COLUMN_STATUS_CODE )
+ return keyspace.prepareQuery( CF_MIGRATION_INFO ).getKey( rowKey ).getColumn( COLUMN_STATUS_CODE )
.execute().getResult().getIntegerValue();
}
//swallow, it doesn't exist
catch ( NotFoundException nfe ) {
- return 0;
+ return -1;
}
catch ( ConnectionException e ) {
throw new DataMigrationException( "Unable to retrieve status", e );
}
}
+
@Override
- public int getCurrentVersion() {
+ public int getSystemVersion() {
try {
- return versionCache.get("currentversion");
- }catch (Exception ee){
- throw new RuntimeException(ee);
+ return keyspace.prepareQuery( CF_MIGRATION_INFO ).getKey( LEGACY_ROW_KEY ).getColumn( COLUMN_VERSION )
+ .execute().getResult().getIntegerValue();
+ }
+ //swallow, it doesn't exist
+ catch ( NotFoundException nfe ) {
+ return -1;
+ }
+ catch ( ConnectionException e ) {
+ throw new DataMigrationException( "Unable to retrieve status", e );
}
- }
-
- @Override
- public void invalidate() {
- versionCache.invalidateAll();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d3f8ee61/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/DataMigration2.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/DataMigration2.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/DataMigration2.java
index f660c02..5830335 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/DataMigration2.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/DataMigration2.java
@@ -41,31 +41,7 @@ public interface DataMigration2<T> {
* Get the version of this migration. It should be unique within the scope of the plugin
* @return
*/
- public int version();
+ public int getVersion();
- public interface ProgressObserver{
- /**
- * Mark the migration as failed
- * @param migrationVersion The migration version running during the failure
- * @param reason The reason to save
- */
- public void failed(final int migrationVersion, final String reason);
-
- /**
- * Mark the migration as failed with a stack trace
- * @param migrationVersion The migration version running during the failure
- * @param reason The error description to save
- * @param throwable The error that happened
- */
- public void failed(final int migrationVersion, final String reason, final Throwable throwable);
-
-
- /**
- * Update the status of the migration with the message
- *
- * @param message The message to save for the status
- */
- public void update(final int migrationVersion, final String message);
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d3f8ee61/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/MigrationPlugin.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/MigrationPlugin.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/MigrationPlugin.java
index f04a035..880cfd1 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/MigrationPlugin.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/MigrationPlugin.java
@@ -24,8 +24,6 @@
package org.apache.usergrid.persistence.core.migration.data.newimpls;
-import org.apache.usergrid.persistence.core.migration.data.DataMigration;
-
/**
* A simple interface to return migration plugins. All versions within this migration plugin should have a name
@@ -34,7 +32,7 @@ public interface MigrationPlugin {
/**
- * Get the name of the plugin
+ * Get the name of the plugin. Must be unique
* @return
*/
public String getName();
@@ -42,6 +40,12 @@ public interface MigrationPlugin {
/**
* Run any migrations that may need to be run in this plugin
*/
- public void run(DataMigration.ProgressObserver observer);
+ public void run(ProgressObserver observer);
+
+ /**
+ * Get the maximum migration version this plugin implements
+ * @return
+ */
+ public int getMaxVersion();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d3f8ee61/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/MigrationRelationship.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/MigrationRelationship.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/MigrationRelationship.java
new file mode 100644
index 0000000..a9ad187
--- /dev/null
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/MigrationRelationship.java
@@ -0,0 +1,41 @@
+/*
+ *
+ * *
+ * * Licensed to the Apache Software Foundation (ASF) under one
+ * * or more contributor license agreements. See the NOTICE file
+ * * distributed with this work for additional information
+ * * regarding copyright ownership. The ASF licenses this file
+ * * to you under the Apache License, Version 2.0 (the
+ * * "License"); you may not use this file except in compliance
+ * * with the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing,
+ * * software distributed under the License is distributed on an
+ * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * * KIND, either express or implied. See the License for the
+ * * specific language governing permissions and limitations
+ * * under the License.
+ * *
+ *
+ */
+
+package org.apache.usergrid.persistence.core.migration.data.newimpls;
+
+
+/**
+ * Simple relationship that defines the current state of the source and destination data versions
+ */
+public class MigrationRelationship<T extends VersionedData> {
+
+ //public so it's FAST. It's also immutable
+ public final T from;
+ public final T to;
+
+
+ public MigrationRelationship( T from, T to ) {
+ this.from = from;
+ this.to = to;
+ }
+}
[44/50] [abbrv] incubator-usergrid git commit: Merge branch
'two-dot-o' into USERGRID-405
Posted by sf...@apache.org.
Merge branch 'two-dot-o' into USERGRID-405
Conflicts:
stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/b35551a0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/b35551a0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/b35551a0
Branch: refs/heads/USERGRID-480
Commit: b35551a07ed539ff8277892c00b7d530722cf784
Parents: 5e6ed4a 61c5f9c
Author: Todd Nine <tn...@apigee.com>
Authored: Thu Mar 19 10:23:12 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Thu Mar 19 10:23:12 2015 -0600
----------------------------------------------------------------------
.../corepersistence/CpRelationManager.java | 7 +-
.../results/FilteringLoader.java | 50 +++++++-----
.../corepersistence/StaleIndexCleanupTest.java | 48 +++++++++++
.../index/impl/EsEntityIndexImpl.java | 83 +++++++++-----------
.../applications/ApplicationsResource.java | 6 --
5 files changed, 121 insertions(+), 73 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b35551a0/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 8804d2b,1838eec..e66bd8d
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@@ -56,9 -56,8 +56,11 @@@ import org.elasticsearch.action.admin.c
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksRequest;
import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksResponse;
+import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse;
+import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
+import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse;
+ import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder;
+ import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse;
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse;
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse;
[23/50] [abbrv] incubator-usergrid git commit: Refactored packages
Posted by sf...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cf32557a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/MigrationDataProvider.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/MigrationDataProvider.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/MigrationDataProvider.java
deleted file mode 100644
index ffe8d9e..0000000
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/MigrationDataProvider.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- *
- * *
- * * Licensed to the Apache Software Foundation (ASF) under one
- * * or more contributor license agreements. See the NOTICE file
- * * distributed with this work for additional information
- * * regarding copyright ownership. The ASF licenses this file
- * * to you under the Apache License, Version 2.0 (the
- * * "License"); you may not use this file except in compliance
- * * with the License. You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing,
- * * software distributed under the License is distributed on an
- * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * * KIND, either express or implied. See the License for the
- * * specific language governing permissions and limitations
- * * under the License.
- * *
- *
- */
-
-package org.apache.usergrid.persistence.core.migration.data.newimpls;
-
-
-import rx.Observable;
-
-
-/**
- * An interface for data providers to implement. The migration must take the migrationdata provider as an argument
- * @param <T>
- */
-public interface MigrationDataProvider<T> {
-
-
- /**
- * Get data that can be used in the migration
- * @return
- */
- public Observable<T> getData();
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cf32557a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/MigrationPlugin.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/MigrationPlugin.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/MigrationPlugin.java
deleted file mode 100644
index 50dc91b..0000000
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/MigrationPlugin.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- *
- * *
- * * Licensed to the Apache Software Foundation (ASF) under one
- * * or more contributor license agreements. See the NOTICE file
- * * distributed with this work for additional information
- * * regarding copyright ownership. The ASF licenses this file
- * * to you under the Apache License, Version 2.0 (the
- * * "License"); you may not use this file except in compliance
- * * with the License. You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing,
- * * software distributed under the License is distributed on an
- * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * * KIND, either express or implied. See the License for the
- * * specific language governing permissions and limitations
- * * under the License.
- * *
- *
- */
-
-package org.apache.usergrid.persistence.core.migration.data.newimpls;
-
-
-
-/**
- * A simple interface to return migration plugins. All versions within this migration plugin should have a name
- */
-public interface MigrationPlugin {
-
-
- /**
- * Get the name of the plugin. Must be unique
- * @return
- */
- public String getName();
-
- /**
- * Run any migrations that may need to be run in this plugin
- */
- public void run(ProgressObserver observer);
-
-
- /**
- * Get the max version this plugin supports
- * @return
- */
- int getMaxVersion();
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cf32557a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/MigrationRelationship.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/MigrationRelationship.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/MigrationRelationship.java
deleted file mode 100644
index 691d8c1..0000000
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/MigrationRelationship.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- *
- * *
- * * Licensed to the Apache Software Foundation (ASF) under one
- * * or more contributor license agreements. See the NOTICE file
- * * distributed with this work for additional information
- * * regarding copyright ownership. The ASF licenses this file
- * * to you under the Apache License, Version 2.0 (the
- * * "License"); you may not use this file except in compliance
- * * with the License. You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing,
- * * software distributed under the License is distributed on an
- * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * * KIND, either express or implied. See the License for the
- * * specific language governing permissions and limitations
- * * under the License.
- * *
- *
- */
-
-package org.apache.usergrid.persistence.core.migration.data.newimpls;
-
-
-/**
- * Simple relationship that defines the current state of the source and destination data versions. Note that
- * ina current system, the from and then to will be the same instance
- */
-public class MigrationRelationship<T extends VersionedData> {
-
- //public so it's FAST. It's also immutable
-
-
- public final T from;
- public final T to;
-
- private final int fromVersion;
- private final int toVersion;
-
-
- public MigrationRelationship( T from, T to ) {
- this.from = from;
- this.to = to;
-
- fromVersion = from.getImplementationVersion();
- toVersion = to.getImplementationVersion();
- }
-
-
- /**
- * Returns true if we need to perform dual writes. IE. the from is not the same as the to
- * @return
- */
- public boolean needsMigration(){
- return fromVersion != toVersion;
- }
-
-
- /**
- * Return true if this is the migration relationship we should use. The version matches the from
- * and is <= the to
- * @param currentVersion
- * @return
- */
- public boolean correctRelationship(final int currentVersion){
- return currentVersion == fromVersion && currentVersion <= toVersion;
- }
-
-
- @Override
- public boolean equals( final Object o ) {
- if ( this == o ) {
- return true;
- }
- if ( !( o instanceof MigrationRelationship ) ) {
- return false;
- }
-
- final MigrationRelationship that = ( MigrationRelationship ) o;
-
- if ( !from.equals( that.from ) ) {
- return false;
- }
- if ( !to.equals( that.to ) ) {
- return false;
- }
-
- return true;
- }
-
-
- @Override
- public int hashCode() {
- int result = from.hashCode();
- result = 31 * result + to.hashCode();
- return result;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cf32557a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/ProgressObserver.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/ProgressObserver.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/ProgressObserver.java
deleted file mode 100644
index 36c721a..0000000
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/ProgressObserver.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- *
- * *
- * * Licensed to the Apache Software Foundation (ASF) under one
- * * or more contributor license agreements. See the NOTICE file
- * * distributed with this work for additional information
- * * regarding copyright ownership. The ASF licenses this file
- * * to you under the Apache License, Version 2.0 (the
- * * "License"); you may not use this file except in compliance
- * * with the License. You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing,
- * * software distributed under the License is distributed on an
- * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * * KIND, either express or implied. See the License for the
- * * specific language governing permissions and limitations
- * * under the License.
- * *
- *
- */
-
-package org.apache.usergrid.persistence.core.migration.data.newimpls;
-
-
-public interface ProgressObserver{
- /**
- * Mark the migration as failed
- * @param migrationVersion The migration version running during the failure
- * @param reason The reason to save
- */
- public void failed(final int migrationVersion, final String reason);
-
- /**
- * Mark the migration as failed with a stack trace
- * @param migrationVersion The migration version running during the failure
- * @param reason The error description to save
- * @param throwable The error that happened
- */
- public void failed(final int migrationVersion, final String reason, final Throwable throwable);
-
-
- /**
- * Update the status of the migration with the message
- *
- * @param message The message to save for the status
- */
- public void update(final int migrationVersion, final String message);
- }
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cf32557a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/VersionedData.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/VersionedData.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/VersionedData.java
deleted file mode 100644
index 14d488b..0000000
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/VersionedData.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- *
- * *
- * * Licensed to the Apache Software Foundation (ASF) under one
- * * or more contributor license agreements. See the NOTICE file
- * * distributed with this work for additional information
- * * regarding copyright ownership. The ASF licenses this file
- * * to you under the Apache License, Version 2.0 (the
- * * "License"); you may not use this file except in compliance
- * * with the License. You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing,
- * * software distributed under the License is distributed on an
- * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * * KIND, either express or implied. See the License for the
- * * specific language governing permissions and limitations
- * * under the License.
- * *
- *
- */
-
-package org.apache.usergrid.persistence.core.migration.data.newimpls;
-
-
-/**
- * Marker interface for implementations that may migrate their internal format
- */
-public interface VersionedData {
-
- /**
- * Get the version of the implementation
- * @return
- */
- public int getImplementationVersion();
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cf32557a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/VersionedMigrationSet.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/VersionedMigrationSet.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/VersionedMigrationSet.java
deleted file mode 100644
index f40f5d0..0000000
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/VersionedMigrationSet.java
+++ /dev/null
@@ -1,153 +0,0 @@
-/*
- *
- * *
- * * Licensed to the Apache Software Foundation (ASF) under one
- * * or more contributor license agreements. See the NOTICE file
- * * distributed with this work for additional information
- * * regarding copyright ownership. The ASF licenses this file
- * * to you under the Apache License, Version 2.0 (the
- * * "License"); you may not use this file except in compliance
- * * with the License. You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing,
- * * software distributed under the License is distributed on an
- * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * * KIND, either express or implied. See the License for the
- * * specific language governing permissions and limitations
- * * under the License.
- * *
- *
- */
-
-package org.apache.usergrid.persistence.core.migration.data.newimpls;
-
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import com.google.common.base.Preconditions;
-
-
-/**
- * A set that represents a set of tuples that are used for
- * @param <T>
- */
-public class VersionedMigrationSet<T extends VersionedData> {
-
-
- /**
- * Cache so that after our initial lookup, it O(1) since this will be used heavily
- *
- */
- private Map<Integer, MigrationRelationship<T>> cacheVersion = new HashMap<>();
-
- private List<MigrationRelationship<T>> orderedVersions = new ArrayList<>();
-
-
- /**
- * Construct this set from a group of tuples. Imagine the following versions
- *
- * v1,
- * v2,
- * v3,
- * v4
- *
- * Migrations can jump from v1->v3, but not directly to v4 without an extraneous migration. This would have 2 relationships
- *
- * v1, v3
- * v2, v3
- * and
- * v3, v4
- *
- *
- * @param migrations
- */
- public VersionedMigrationSet( final MigrationRelationship<T>... migrations ){
- Preconditions.checkNotNull(migrations, "versions must not be null");
- Preconditions.checkArgument( migrations.length > 0, "You must specify at least 1 migrationrelationship" );
-
- orderedVersions.addAll( Arrays.asList(migrations ) );
-
- Collections.sort( orderedVersions, new VersionedDataComparator() );
-
- }
-
-
- /**
- * Get the migration relationship based on our current version. This will return a range that includes the current
- * system version as the source, and the highest version we can roll to in the to field
- * @param currentVersion
- * @return The MigrationRelationship. Note the from and the to could be the same version in a current system.
- */
- public MigrationRelationship<T> getMigrationRelationship( final int currentVersion ){
-
- final MigrationRelationship<T> relationship = cacheVersion.get( currentVersion );
-
- if(relationship != null){
- return relationship;
- }
-
- //not there, find it. Not the most efficient, but it happens once per version, which rarely changes, so not a big deal
-
-
- for(MigrationRelationship<T> current: orderedVersions){
-
- //not our instance, the from is too high
- //our from is this instance, so we support this tuple. Our future is >= as well, so we can perform this I/O
- if ( current.correctRelationship( currentVersion )) {
- cacheVersion.put( currentVersion, current );
- return current;
- }
-
- }
-
- //if we get here, something is wrong
- throw new IllegalArgumentException( "Could not find a migration version for version " + currentVersion + " min found was " + orderedVersions.get( orderedVersions.size()-1 ) );
-
-
- }
-
-
- /**
- * Given the current system version, return the maximum migration version we can move to
- * @param currentVersion
- * @return
- */
- public int getMaxVersion(final int currentVersion){
- return getMigrationRelationship( currentVersion ).to.getImplementationVersion();
- }
-
-
-
- /**
- * Orders from high to low
- */
- private final class VersionedDataComparator implements Comparator<MigrationRelationship<T>>
- {
-
- @Override
- public int compare( final MigrationRelationship<T> o1, final MigrationRelationship<T> o2 ) {
- //Put higher target version first, since that's what we want to match based on current state and source
-
- //order by the source. Put highest source first
- int compare = Integer.compare( o1.to.getImplementationVersion(), o2.to.getImplementationVersion() ) *-1;
-
-
- //put higher from first, if we fall within a range here we're good
- if(compare == 0){
- compare = Integer.compare( o1.from.getImplementationVersion(), o2.from.getImplementationVersion() ) *-1;
- }
-
- return compare;
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cf32557a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManagerImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManagerImplTest.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManagerImplTest.java
index a6d5717..5d516d4 100644
--- a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManagerImplTest.java
+++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManagerImplTest.java
@@ -1,248 +1,148 @@
/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * * Licensed to the Apache Software Foundation (ASF) under one
- * * or more contributor license agreements. See the NOTICE file
- * * distributed with this work for additional information
- * * regarding copyright ownership. The ASF licenses this file
- * * to you under the Apache License, Version 2.0 (the
- * * "License"); you may not use this file except in compliance
- * * with the License. You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing,
- * * software distributed under the License is distributed on an
- * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * * KIND, either express or implied. See the License for the
- * * specific language governing permissions and limitations
- * * under the License.
+ * http://www.apache.org/licenses/LICENSE-2.0
*
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
*/
package org.apache.usergrid.persistence.core.migration.data;
+import java.util.HashSet;
+import java.util.Set;
+
import org.junit.Test;
-import static org.junit.Assert.fail;
+import org.apache.usergrid.persistence.core.migration.schema.MigrationException;
+
+import static org.junit.Assert.*;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
-/**
- * Tests our data migration manager
- */
public class DataMigrationManagerImplTest {
- //TODO USERGRID-405 fix this
-//
-// AllEntitiesInSystemObservable allEntitiesInSystemObservable = new AllEntitiesInSystemObservable() {
-// @Override
-// public Observable<ApplicationEntityGroup> getAllEntitiesInSystem(int bufferSize) {
-//
-// return Observable.create(new Observable.OnSubscribe<ApplicationEntityGroup>() {
-// @Override
-// public void call(Subscriber<? super ApplicationEntityGroup> subscriber) {
-// ApplicationEntityGroup entityGroup = new ApplicationEntityGroup(mock(ApplicationScope.class),new ArrayList<EntityIdScope>());
-// subscriber.onNext(entityGroup);
-// subscriber.onCompleted();
-// }
-// });
-// }
-//
-// @Override
-// public Observable<ApplicationEntityGroup> getAllEntitiesInSystem(Observable appIdObservable, int bufferSize) {
-// return this.getAllEntitiesInSystem(bufferSize) ;
-// }
-// };
-//
-// ApplicationObservable allApplicationsObservable = new ApplicationObservable() {
-// @Override
-// public Observable<Id> getAllApplicationIds() {
-// return Observable.just( (Id)new SimpleId("application"));
-// }
-//
-// @Override
-// public Observable<ApplicationScope> getAllApplicationScopes() {
-// return Observable.just( (ApplicationScope)new ApplicationScopeImpl((Id)new SimpleId("application")));
-// }
-// };
-//
-// @Test
-// public void noMigrations() throws MigrationException {
-// final MigrationInfoSerialization serialization = mock( MigrationInfoSerialization.class );
-// when(serialization.getCurrentVersion()).thenReturn(1);
-//
-// Set<DataMigration> emptyMigration = new HashSet<>();
-//
-// DataMigrationManagerImpl migrationManager = new DataMigrationManagerImpl( serialization, emptyMigration, allEntitiesInSystemObservable,allApplicationsObservable );
-//
-// migrationManager.migrate();
-//
-// verify( serialization, never() ).setStatusMessage( any( String.class ) );
-// verify( serialization, never() ).setStatusCode( any( Integer.class ) );
-// verify( serialization, never() ).setVersion( any( Integer.class ) );
-// }
-//
-//
-// @Test
-// public void multipleMigrations() throws Throwable {
-// final MigrationInfoSerialization serialization = mock( MigrationInfoSerialization.class );
-// when(serialization.getCurrentVersion()).thenReturn(1);
-//
-//
-// final ApplicationDataMigration v1 = mock( ApplicationDataMigration.class );
-// when( v1.getVersion() ).thenReturn( 2 );
-// when( v1.migrate(any(Observable.class), any(DataMigration.ProgressObserver.class))).thenReturn(Observable.empty());
-//
-// final ApplicationDataMigration v2 = mock( ApplicationDataMigration.class );
-// when( v2.getVersion() ).thenReturn( 3 );
-// when(v2.migrate(any(Observable.class), any(DataMigration.ProgressObserver.class))).thenReturn(Observable.empty());
-//
-//
-// Set<DataMigration> migrations = new HashSet<>();
-// migrations.add( v1 );
-// migrations.add( v2 );
-//
-//
-//
-// DataMigrationManagerImpl migrationManager = new DataMigrationManagerImpl( serialization, migrations,allEntitiesInSystemObservable,allApplicationsObservable );
-//
-// migrationManager.migrate();
-//
-//
-// verify( v1 ).migrate(any(Observable.class), any( DataMigration.ProgressObserver.class ) );
-// verify( v2 ).migrate(any(Observable.class), any( DataMigration.ProgressObserver.class ) );
-//
-// //verify we set the running status
-// verify( serialization, times( 2 ) ).setStatusCode( DataMigrationManagerImpl.StatusCode.RUNNING.status );
-//
-// //set the status message
-// verify( serialization, times( 2 * 2 ) ).setStatusMessage( any( String.class ) );
-//
-// verify( serialization ).setStatusCode( DataMigrationManagerImpl.StatusCode.COMPLETE.status );
-//
-// //verify we set version 1
-// verify( serialization ).setVersion( 2 );
-//
-// //verify we set version 2
-// verify( serialization ).setVersion( 3 );
-// }
-//
-//
-// @Test
-// public void shortCircuitVersionFails() throws Throwable {
-// final MigrationInfoSerialization serialization = mock( MigrationInfoSerialization.class );
-// when(serialization.getCurrentVersion()).thenReturn(1);
-//
-//
-// final ApplicationDataMigration v1 = mock( ApplicationDataMigration.class,"mock1" );
-// when( v1.getVersion() ).thenReturn( 2 );
-// when( v1.getType() ).thenReturn(DataMigration.MigrationType.Entities);
-// when( v1.migrate(any(Observable.class), any(DataMigration.ProgressObserver.class))).thenReturn(Observable.empty());
-//
-// //throw an exception
-// when( v1.migrate(any(Observable.class),
-// any(DataMigration.ProgressObserver.class) )).thenThrow(new RuntimeException( "Something bad happened" ));
-//
-// final ApplicationDataMigration v2 = mock( ApplicationDataMigration.class,"mock2" );
-// when( v2.getType() ).thenReturn(DataMigration.MigrationType.Entities);
-// when( v2.getVersion() ).thenReturn( 3 );
-//
-// Set<DataMigration> migrations = new HashSet<>();
-// migrations.add( v1 );
-// migrations.add( v2 );
-//
-// DataMigrationManagerImpl migrationManager
-// = new DataMigrationManagerImpl( serialization, migrations,allEntitiesInSystemObservable,allApplicationsObservable );
-//
-// migrationManager.migrate();
-//
-//
-// verify( v1 ).migrate( any(Observable.class),any( DataMigration.ProgressObserver.class ) );
-//
-// //verify we don't run migration
-// verify( v2, never() ).migrate( any(Observable.class),any( DataMigration.ProgressObserver.class ) );
-//
-// //verify we set the running status
-// verify( serialization, times( 1 ) ).setStatusCode( DataMigrationManagerImpl.StatusCode.RUNNING.status );
-//
-// //set the status message
-// verify( serialization, times( 2 ) ).setStatusMessage( any( String.class ) );
-//
-// //verify we set an error
-// verify( serialization ).setStatusCode( DataMigrationManagerImpl.StatusCode.ERROR.status );
-//
-// //verify we never set version 1
-// verify( serialization, never() ).setVersion( 1 );
-//
-// //verify we never set version 2
-// verify( serialization, never() ).setVersion( 2 );
-// }
-//
-//
-// @Test
-// public void failStopsProgress() throws Throwable {
-// final MigrationInfoSerialization serialization = mock(MigrationInfoSerialization.class);
-// when(serialization.getCurrentVersion()).thenReturn(1);
-//
-// final CollectionDataMigration v1 = mock( CollectionDataMigration.class );
-// when( v1.getVersion() ).thenReturn( 2 );
-// when( v1.getType() ).thenReturn(DataMigration.MigrationType.Entities);
-// when( v1.migrate(any(Observable.class), any(DataMigration.ProgressObserver.class))).thenReturn(Observable.empty());
-//
-// final int returnedCode = 100;
-//
-// final String reason = "test reason";
-//
-// //mark as fail but don't
-// when(v1.migrate(any(Observable.class), any(DataMigration.ProgressObserver.class))).thenAnswer(
-// new Answer<Object>() {
-// @Override
-// public Object answer(final InvocationOnMock invocation) throws Throwable {
-// final DataMigration.ProgressObserver progressObserver =
-// (DataMigration.ProgressObserver) invocation.getArguments()[1];
-//
-// progressObserver.failed(returnedCode, reason);
-// return null;
-// }
-// }
-//
-// );
-//
-// final CollectionDataMigration v2 = mock( CollectionDataMigration.class );
-// when( v2.getVersion() ).thenReturn( 3 );
-// when( v2.getType() ).thenReturn(DataMigration.MigrationType.Entities);
-// when(v2.migrate(any(Observable.class), any(DataMigration.ProgressObserver.class))).thenReturn(Observable.empty());
-//
-// Set<DataMigration> applicationDataMigrations = new HashSet<>();
-// applicationDataMigrations.add( v1 );
-// applicationDataMigrations.add(v2);
-//
-//
-// DataMigrationManagerImpl migrationManager = new DataMigrationManagerImpl( serialization, applicationDataMigrations,allEntitiesInSystemObservable, allApplicationsObservable );
-//
-// migrationManager.migrate();
-//
-//
-// verify( v1 ).migrate(any(Observable.class), any( DataMigration.ProgressObserver.class ) );
-//
-// //verify we don't run migration
-// verify( v2, never() ).migrate( any(Observable.class),any( DataMigration.ProgressObserver.class ) );
-//
-// //verify we set the running status
-// verify( serialization, times( 1 ) ).setStatusCode( DataMigrationManagerImpl.StatusCode.RUNNING.status );
-//
-// //set the status message
-// verify( serialization ).setStatusMessage( "Migration version 2. Starting migration" );
-//
-// verify( serialization ).setStatusMessage( "Migration version 100. Failed to migrate, reason is appended. Error 'test reason'" );
-//
-// //verify we set an error
-// verify( serialization, times(2) ).setStatusCode( DataMigrationManagerImpl.StatusCode.ERROR.status );
-//
-// //verify we never set version 1
-// verify( serialization, never() ).setVersion( 1 );
-//
-// //verify we never set version 2
-// verify( serialization, never() ).setVersion( 2 );
-// }
+
+ @Test
+ public void testNoPlugins(){
+
+ final Set<MigrationPlugin> plugins = new HashSet<>( );
+
+ final MigrationInfoSerialization migrationInfoSerialization = mock(MigrationInfoSerialization.class);
+
+
+ DataMigrationManagerImpl migrationManager = new DataMigrationManagerImpl( plugins, migrationInfoSerialization );
+
+
+ Set<String> pluginNames = migrationManager.getPluginNames();
+
+ assertEquals(0, pluginNames.size());
+
+ }
+
+
+
+ @Test
+ public void test2Plugins() throws MigrationException {
+
+ final Set<MigrationPlugin> plugins = new HashSet<>( );
+
+ MigrationPlugin plugin1 = mock(MigrationPlugin.class);
+
+ when(plugin1.getName()).thenReturn("plugin1");
+
+ MigrationPlugin plugin2 = mock(MigrationPlugin.class);
+
+ when(plugin2.getName()).thenReturn("plugin2");
+
+ plugins.add( plugin1 );
+ plugins.add( plugin2 );
+
+
+
+ final MigrationInfoSerialization migrationInfoSerialization = mock(MigrationInfoSerialization.class);
+
+
+ DataMigrationManagerImpl migrationManager = new DataMigrationManagerImpl( plugins, migrationInfoSerialization );
+
+
+ Set<String> pluginNames = migrationManager.getPluginNames();
+
+ assertEquals(2, pluginNames.size());
+
+ assertTrue(pluginNames.contains( "plugin1"));
+
+ assertTrue(pluginNames.contains( "plugin2" ));
+
+ //now run them
+
+ migrationManager.migrate();
+
+ verify(plugin1).run( any(ProgressObserver.class) );
+
+ verify(plugin2).run( any(ProgressObserver.class) );
+
+ }
+
+
+
+
+ @Test
+ public void testRunning() throws MigrationException {
+
+ final Set<MigrationPlugin> plugins = new HashSet<>( );
+
+ MigrationPlugin plugin1 = mock(MigrationPlugin.class);
+
+ when(plugin1.getName()).thenReturn("plugin1");
+
+ plugins.add( plugin1 );
+
+
+
+ final MigrationInfoSerialization migrationInfoSerialization = mock(MigrationInfoSerialization.class);
+
+ when(migrationInfoSerialization.getStatusCode( "plugin1" )).thenReturn( DataMigrationManagerImpl.StatusCode.RUNNING.status );
+
+
+
+
+ DataMigrationManagerImpl migrationManager = new DataMigrationManagerImpl( plugins, migrationInfoSerialization );
+
+
+ boolean status = migrationManager.isRunning();
+
+ assertTrue("Status is set", status);
+
+
+ when(migrationInfoSerialization.getStatusCode( "plugin1" )).thenReturn( DataMigrationManagerImpl.StatusCode.COMPLETE.status );
+
+ status = migrationManager.isRunning();
+
+ assertFalse( "Status is not running", status );
+
+
+ when(migrationInfoSerialization.getStatusCode( "plugin1" )).thenReturn( DataMigrationManagerImpl.StatusCode.ERROR.status );
+
+ status = migrationManager.isRunning();
+
+ assertFalse("Status is not running", status);
+ }
+
+
+
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cf32557a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/MigrationInfoSerializationTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/MigrationInfoSerializationTest.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/MigrationInfoSerializationTest.java
index 7ac1fb2..1b6dd3f 100644
--- a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/MigrationInfoSerializationTest.java
+++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/MigrationInfoSerializationTest.java
@@ -22,18 +22,16 @@
package org.apache.usergrid.persistence.core.migration.data;
-
-import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.RunWith;
-import org.apache.usergrid.persistence.core.migration.schema.MigrationException;
-import org.apache.usergrid.persistence.core.migration.schema.MigrationManager;
-import org.apache.usergrid.persistence.core.test.ITRunner;
import org.apache.usergrid.persistence.core.guice.MigrationManagerRule;
import org.apache.usergrid.persistence.core.guice.TestCommonModule;
+import org.apache.usergrid.persistence.core.migration.schema.MigrationException;
+import org.apache.usergrid.persistence.core.migration.schema.MigrationManager;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.core.test.ITRunner;
import org.apache.usergrid.persistence.core.test.UseModules;
import com.google.inject.Inject;
@@ -42,16 +40,12 @@ import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
-import static org.junit.Assert.fail;
@RunWith( ITRunner.class )
@UseModules( TestCommonModule.class )
-@Ignore()
public class MigrationInfoSerializationTest {
- //TODO USERGRID-405 fix this
-
@Inject
@Rule
public MigrationManagerRule migrationManagerRule;
@@ -68,53 +62,55 @@ public class MigrationInfoSerializationTest {
@Inject
protected MigrationInfoSerialization migrationInfoSerialization;
-//
-// @Test
-// public void basicTest() throws ConnectionException, MigrationException {
-//
-// //drop the column family, then run setup
-// keyspace.dropColumnFamily( MigrationInfoSerializationImpl.CF_MIGRATION_INFO.getName() );
-//
-// migrationManager.migrate();
-//
-// //test getting nothing works
-// final String emptyStatus = migrationInfoSerialization.getStatusMessage();
-//
-// assertNull(emptyStatus);
-//
-// final int unsavedVersion = migrationInfoSerialization.getVersion();
-//
-// assertEquals(0, unsavedVersion);
-//
-// final int statusCode = migrationInfoSerialization.getStatusCode();
-//
-// assertEquals(0, statusCode);
-//
-// //now update them
-//
-// final String savedStatus = "I'm a test status";
-//
-// migrationInfoSerialization.setStatusMessage( savedStatus );
-//
-// final String returnedStatus = migrationInfoSerialization.getStatusMessage();
-//
-// assertEquals("Correct status returned", savedStatus, returnedStatus);
-//
-//
-// final int savedVersion = 100;
-//
-// migrationInfoSerialization.setVersion( savedVersion );
-//
-// final int returnedVersion = migrationInfoSerialization.getVersion();
-//
-// assertEquals("Correct version returned", savedVersion, returnedVersion);
-//
-// final int savedStatusCode = 200;
-//
-// migrationInfoSerialization.setStatusCode( savedStatusCode );
-//
-// final int returnedStatusCode = migrationInfoSerialization.getStatusCode();
-//
-// assertEquals("Status code was set correctly", savedStatusCode, returnedStatusCode);
-// }
+
+ @Test
+ public void basicTest() throws ConnectionException, MigrationException {
+
+ final String migrationModule = "test";
+
+ //drop the column family, then run setup
+ keyspace.dropColumnFamily( MigrationInfoSerializationImpl.CF_MIGRATION_INFO.getName() );
+
+ migrationManager.migrate();
+
+ //test getting nothing works
+ final String emptyStatus = migrationInfoSerialization.getStatusMessage(migrationModule);
+
+ assertNull(emptyStatus);
+
+ final int unsavedVersion = migrationInfoSerialization.getVersion(migrationModule);
+
+ assertEquals(0, unsavedVersion);
+
+ final int statusCode = migrationInfoSerialization.getStatusCode(migrationModule);
+
+ assertEquals(0, statusCode);
+
+ //now update them
+
+ final String savedStatus = "I'm a test status";
+
+ migrationInfoSerialization.setStatusMessage(migrationModule, savedStatus );
+
+ final String returnedStatus = migrationInfoSerialization.getStatusMessage(migrationModule);
+
+ assertEquals("Correct status returned", savedStatus, returnedStatus);
+
+
+ final int savedVersion = 100;
+
+ migrationInfoSerialization.setVersion(migrationModule, savedVersion );
+
+ final int returnedVersion = migrationInfoSerialization.getVersion(migrationModule);
+
+ assertEquals("Correct version returned", savedVersion, returnedVersion);
+
+ final int savedStatusCode = 200;
+
+ migrationInfoSerialization.setStatusCode(migrationModule, savedStatusCode );
+
+ final int returnedStatusCode = migrationInfoSerialization.getStatusCode(migrationModule);
+
+ assertEquals("Status code was set correctly", savedStatusCode, returnedStatusCode);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cf32557a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/TestMigrationDataProvider.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/TestMigrationDataProvider.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/TestMigrationDataProvider.java
new file mode 100644
index 0000000..61e5207
--- /dev/null
+++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/TestMigrationDataProvider.java
@@ -0,0 +1,61 @@
+/*
+ *
+ * *
+ * * Licensed to the Apache Software Foundation (ASF) under one
+ * * or more contributor license agreements. See the NOTICE file
+ * * distributed with this work for additional information
+ * * regarding copyright ownership. The ASF licenses this file
+ * * to you under the Apache License, Version 2.0 (the
+ * * "License"); you may not use this file except in compliance
+ * * with the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing,
+ * * software distributed under the License is distributed on an
+ * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * * KIND, either express or implied. See the License for the
+ * * specific language governing permissions and limitations
+ * * under the License.
+ * *
+ *
+ */
+
+package org.apache.usergrid.persistence.core.migration.data;
+
+
+import org.apache.usergrid.persistence.core.migration.data.MigrationDataProvider;
+
+import rx.Observable;
+
+
+/**
+ * A simple test class that will emit the provided test data when subscribed
+ * @param <T>
+ */
+public class TestMigrationDataProvider<T> implements MigrationDataProvider<T> {
+
+
+
+ //default to nothing so that we don't return null
+ private Observable<T> observable = Observable.empty();
+
+
+ public TestMigrationDataProvider( ) {}
+
+
+ @Override
+ public Observable<T> getData() {
+ return observable;
+ }
+
+
+ /**
+ * Set this observable to return when invoked
+ *
+ * @param observable
+ */
+ public void setObservable( final Observable<T> observable ) {
+ this.observable = observable;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cf32557a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/TestProgressObserver.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/TestProgressObserver.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/TestProgressObserver.java
index 63fad42..1612779 100644
--- a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/TestProgressObserver.java
+++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/TestProgressObserver.java
@@ -23,18 +23,30 @@ package org.apache.usergrid.persistence.core.migration.data;
import java.util.ArrayList;
import java.util.List;
-import org.apache.usergrid.persistence.core.migration.data.newimpls.ProgressObserver;
-
public class TestProgressObserver implements ProgressObserver {
private boolean failed = false;
+ private boolean started = false;
+
private List<String> updates = new ArrayList<>( 100 );
@Override
+ public void start() {
+ started = true;
+ }
+
+
+ @Override
+ public void stop() {
+ started = false;
+ }
+
+
+ @Override
public void failed( final int migrationVersion, final String reason ) {
failed = true;
}
@@ -56,11 +68,17 @@ public class TestProgressObserver implements ProgressObserver {
* Get if we failed
* @return
*/
- public boolean getFailed() {
+ public boolean isFailed() {
return failed;
}
+ public boolean isStarted() {
+ return started;
+ }
+
+
+
/**
* Get update messages
* @return
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cf32557a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/VersionedMigrationSetTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/VersionedMigrationSetTest.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/VersionedMigrationSetTest.java
new file mode 100644
index 0000000..fc23cb4
--- /dev/null
+++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/VersionedMigrationSetTest.java
@@ -0,0 +1,198 @@
+/*
+ *
+ * *
+ * * Licensed to the Apache Software Foundation (ASF) under one
+ * * or more contributor license agreements. See the NOTICE file
+ * * distributed with this work for additional information
+ * * regarding copyright ownership. The ASF licenses this file
+ * * to you under the Apache License, Version 2.0 (the
+ * * "License"); you may not use this file except in compliance
+ * * with the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing,
+ * * software distributed under the License is distributed on an
+ * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * * KIND, either express or implied. See the License for the
+ * * specific language governing permissions and limitations
+ * * under the License.
+ * *
+ *
+ */
+
+package org.apache.usergrid.persistence.core.migration.data;
+
+
+import org.junit.Test;
+
+import org.apache.usergrid.persistence.core.migration.data.MigrationRelationship;
+import org.apache.usergrid.persistence.core.migration.data.VersionedData;
+import org.apache.usergrid.persistence.core.migration.data.VersionedMigrationSet;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+
+public class VersionedMigrationSetTest {
+
+
+ /**
+ * a single version that goes from 1 to 3, and 2 to 3. then current as 3, 3. 1, 2 and 3, should have this in
+ * their range
+ */
+ @Test
+ public void singleVersion() {
+ //
+ final MigrationRelationship<TestVersionImpl> relationship1_3 =
+ new MigrationRelationship<>( new TestVersionImpl( 1 ), new TestVersionImpl( 3 ) );
+
+ final MigrationRelationship<TestVersionImpl> relationship2_3 =
+ new MigrationRelationship<>( new TestVersionImpl( 2 ), new TestVersionImpl( 3 ) );
+
+ //our current state, a necessity based on the data structure
+
+ final MigrationRelationship<TestVersionImpl> current =
+ new MigrationRelationship<>( new TestVersionImpl( 3 ), new TestVersionImpl( 3 ) );
+
+
+ final VersionedMigrationSet<TestVersionImpl> set =
+ new VersionedMigrationSet<>( relationship1_3, relationship2_3, current );
+
+
+ try {
+ set.getMigrationRelationship( 0 );
+ fail( "I should throw an exception" );
+ }
+ catch ( IllegalArgumentException iae ) {
+ //swallow, it's outside the range
+ }
+
+ MigrationRelationship<TestVersionImpl> migrationRelationship = set.getMigrationRelationship( 1 );
+
+ assertEquals( relationship1_3, migrationRelationship );
+
+
+ migrationRelationship = set.getMigrationRelationship( 2 );
+
+ assertEquals( relationship2_3, migrationRelationship );
+
+ migrationRelationship = set.getMigrationRelationship( 3 );
+
+ assertEquals( current, migrationRelationship );
+
+ try {
+ set.getMigrationRelationship( 4 );
+ fail( "I should throw an exception" );
+ }
+ catch ( IllegalArgumentException iae ) {
+ //swallow, it's outside the range
+ }
+ }
+
+
+ /**
+ * a single version that goes from 1 to 3. versions that go from 2 to 3, then a barrier migration that must be run.
+ * This can happen during a massive data change. From there we can move on from 3 forward, so we go from 3:6, 4:6,
+ * 5:6, and 6:6, This should mean an older system say on v2, can jump from v2 to v3, then from v3 directly to v6.
+ */
+ @Test
+ public void versionBounds() {
+ //
+ final MigrationRelationship<TestVersionImpl> relationship1_3 =
+ new MigrationRelationship<>( new TestVersionImpl( 1 ), new TestVersionImpl( 3 ) );
+
+ final MigrationRelationship<TestVersionImpl> relationship2_3 =
+ new MigrationRelationship<>( new TestVersionImpl( 2 ), new TestVersionImpl( 3 ) );
+
+ //our current state, a necessity based on the data structure
+ final MigrationRelationship<TestVersionImpl> relationship3_6 =
+ new MigrationRelationship<>( new TestVersionImpl( 3 ), new TestVersionImpl( 6 ) );
+
+
+ final MigrationRelationship<TestVersionImpl> relationship4_6 =
+ new MigrationRelationship<>( new TestVersionImpl( 4 ), new TestVersionImpl( 6 ) );
+
+
+ final MigrationRelationship<TestVersionImpl> relationship5_6 =
+ new MigrationRelationship<>( new TestVersionImpl( 5 ), new TestVersionImpl( 6 ) );
+
+
+ final MigrationRelationship<TestVersionImpl> current =
+ new MigrationRelationship<>( new TestVersionImpl( 6 ), new TestVersionImpl( 6 ) );
+
+
+ final VersionedMigrationSet<TestVersionImpl> set =
+ new VersionedMigrationSet<>( relationship1_3, relationship2_3, relationship3_6, relationship4_6,
+ relationship5_6, current );
+
+
+ try {
+ set.getMigrationRelationship( 0 );
+ fail( "I should throw an exception" );
+ }
+ catch ( IllegalArgumentException iae ) {
+ //swallow, it's outside the range
+ }
+
+ MigrationRelationship<TestVersionImpl> migrationRelationship = set.getMigrationRelationship( 1 );
+
+ assertEquals( relationship1_3, migrationRelationship );
+
+ migrationRelationship = set.getMigrationRelationship( 2 );
+
+ assertEquals( relationship2_3, migrationRelationship );
+
+
+ //now go from v3, we should get 3 to 6
+
+ migrationRelationship = set.getMigrationRelationship( 3 );
+
+ assertEquals( relationship3_6, migrationRelationship );
+
+ migrationRelationship = set.getMigrationRelationship( 4 );
+
+ assertEquals( relationship4_6, migrationRelationship );
+
+ migrationRelationship = set.getMigrationRelationship( 5 );
+
+ assertEquals( relationship5_6, migrationRelationship );
+
+ migrationRelationship = set.getMigrationRelationship( 6 );
+
+ assertEquals( current, migrationRelationship );
+
+
+ try {
+ set.getMigrationRelationship( 7 );
+ fail( "I should throw an exception" );
+ }
+ catch ( IllegalArgumentException iae ) {
+ //swallow, it's outside the range
+ }
+ }
+
+
+ @Test( expected = IllegalArgumentException.class )
+ public void testNoInput() {
+ new VersionedMigrationSet<TestVersionImpl>();
+ }
+
+
+ /**
+ * Create the test version impl. Just returns the version provided
+ */
+ private static final class TestVersionImpl implements VersionedData {
+
+ private final int version;
+
+
+ private TestVersionImpl( final int version ) {this.version = version;}
+
+
+ @Override
+ public int getImplementationVersion() {
+ return version;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cf32557a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/newimpls/TestMigrationDataProvider.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/newimpls/TestMigrationDataProvider.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/newimpls/TestMigrationDataProvider.java
deleted file mode 100644
index 9e99c6f..0000000
--- a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/newimpls/TestMigrationDataProvider.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- *
- * *
- * * Licensed to the Apache Software Foundation (ASF) under one
- * * or more contributor license agreements. See the NOTICE file
- * * distributed with this work for additional information
- * * regarding copyright ownership. The ASF licenses this file
- * * to you under the Apache License, Version 2.0 (the
- * * "License"); you may not use this file except in compliance
- * * with the License. You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing,
- * * software distributed under the License is distributed on an
- * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * * KIND, either express or implied. See the License for the
- * * specific language governing permissions and limitations
- * * under the License.
- * *
- *
- */
-
-package org.apache.usergrid.persistence.core.migration.data.newimpls;
-
-
-import java.util.Collection;
-
-import rx.Observable;
-
-
-/**
- * A simple test class that will emit the provided test data when subscribed
- * @param <T>
- */
-public class TestMigrationDataProvider<T> implements MigrationDataProvider<T> {
-
-
-
- //default to nothing so that we don't return null
- private Observable<T> observable = Observable.empty();
-
-
- public TestMigrationDataProvider( ) {}
-
-
- @Override
- public Observable<T> getData() {
- return observable;
- }
-
-
- /**
- * Set this observable to return when invoked
- *
- * @param observable
- */
- public void setObservable( final Observable<T> observable ) {
- this.observable = observable;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cf32557a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/newimpls/VersionedMigrationSetTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/newimpls/VersionedMigrationSetTest.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/newimpls/VersionedMigrationSetTest.java
deleted file mode 100644
index e35409a..0000000
--- a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/newimpls/VersionedMigrationSetTest.java
+++ /dev/null
@@ -1,194 +0,0 @@
-/*
- *
- * *
- * * Licensed to the Apache Software Foundation (ASF) under one
- * * or more contributor license agreements. See the NOTICE file
- * * distributed with this work for additional information
- * * regarding copyright ownership. The ASF licenses this file
- * * to you under the Apache License, Version 2.0 (the
- * * "License"); you may not use this file except in compliance
- * * with the License. You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing,
- * * software distributed under the License is distributed on an
- * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * * KIND, either express or implied. See the License for the
- * * specific language governing permissions and limitations
- * * under the License.
- * *
- *
- */
-
-package org.apache.usergrid.persistence.core.migration.data.newimpls;
-
-
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-
-
-public class VersionedMigrationSetTest {
-
-
- /**
- * a single version that goes from 1 to 3, and 2 to 3. then current as 3, 3. 1, 2 and 3, should have this in
- * their range
- */
- @Test
- public void singleVersion() {
- //
- final MigrationRelationship<TestVersionImpl> relationship1_3 =
- new MigrationRelationship<>( new TestVersionImpl( 1 ), new TestVersionImpl( 3 ) );
-
- final MigrationRelationship<TestVersionImpl> relationship2_3 =
- new MigrationRelationship<>( new TestVersionImpl( 2 ), new TestVersionImpl( 3 ) );
-
- //our current state, a necessity based on the data structure
-
- final MigrationRelationship<TestVersionImpl> current =
- new MigrationRelationship<>( new TestVersionImpl( 3 ), new TestVersionImpl( 3 ) );
-
-
- final VersionedMigrationSet<TestVersionImpl> set =
- new VersionedMigrationSet<>( relationship1_3, relationship2_3, current );
-
-
- try {
- set.getMigrationRelationship( 0 );
- fail( "I should throw an exception" );
- }
- catch ( IllegalArgumentException iae ) {
- //swallow, it's outside the range
- }
-
- MigrationRelationship<TestVersionImpl> migrationRelationship = set.getMigrationRelationship( 1 );
-
- assertEquals( relationship1_3, migrationRelationship );
-
-
- migrationRelationship = set.getMigrationRelationship( 2 );
-
- assertEquals( relationship2_3, migrationRelationship );
-
- migrationRelationship = set.getMigrationRelationship( 3 );
-
- assertEquals( current, migrationRelationship );
-
- try {
- set.getMigrationRelationship( 4 );
- fail( "I should throw an exception" );
- }
- catch ( IllegalArgumentException iae ) {
- //swallow, it's outside the range
- }
- }
-
-
- /**
- * a single version that goes from 1 to 3. versions that go from 2 to 3, then a barrier migration that must be run.
- * This can happen during a massive data change. From there we can move on from 3 forward, so we go from 3:6, 4:6,
- * 5:6, and 6:6, This should mean an older system say on v2, can jump from v2 to v3, then from v3 directly to v6.
- */
- @Test
- public void versionBounds() {
- //
- final MigrationRelationship<TestVersionImpl> relationship1_3 =
- new MigrationRelationship<>( new TestVersionImpl( 1 ), new TestVersionImpl( 3 ) );
-
- final MigrationRelationship<TestVersionImpl> relationship2_3 =
- new MigrationRelationship<>( new TestVersionImpl( 2 ), new TestVersionImpl( 3 ) );
-
- //our current state, a necessity based on the data structure
- final MigrationRelationship<TestVersionImpl> relationship3_6 =
- new MigrationRelationship<>( new TestVersionImpl( 3 ), new TestVersionImpl( 6 ) );
-
-
- final MigrationRelationship<TestVersionImpl> relationship4_6 =
- new MigrationRelationship<>( new TestVersionImpl( 4 ), new TestVersionImpl( 6 ) );
-
-
- final MigrationRelationship<TestVersionImpl> relationship5_6 =
- new MigrationRelationship<>( new TestVersionImpl( 5 ), new TestVersionImpl( 6 ) );
-
-
- final MigrationRelationship<TestVersionImpl> current =
- new MigrationRelationship<>( new TestVersionImpl( 6 ), new TestVersionImpl( 6 ) );
-
-
- final VersionedMigrationSet<TestVersionImpl> set =
- new VersionedMigrationSet<>( relationship1_3, relationship2_3, relationship3_6, relationship4_6,
- relationship5_6, current );
-
-
- try {
- set.getMigrationRelationship( 0 );
- fail( "I should throw an exception" );
- }
- catch ( IllegalArgumentException iae ) {
- //swallow, it's outside the range
- }
-
- MigrationRelationship<TestVersionImpl> migrationRelationship = set.getMigrationRelationship( 1 );
-
- assertEquals( relationship1_3, migrationRelationship );
-
- migrationRelationship = set.getMigrationRelationship( 2 );
-
- assertEquals( relationship2_3, migrationRelationship );
-
-
- //now go from v3, we should get 3 to 6
-
- migrationRelationship = set.getMigrationRelationship( 3 );
-
- assertEquals( relationship3_6, migrationRelationship );
-
- migrationRelationship = set.getMigrationRelationship( 4 );
-
- assertEquals( relationship4_6, migrationRelationship );
-
- migrationRelationship = set.getMigrationRelationship( 5 );
-
- assertEquals( relationship5_6, migrationRelationship );
-
- migrationRelationship = set.getMigrationRelationship( 6 );
-
- assertEquals( current, migrationRelationship );
-
-
- try {
- set.getMigrationRelationship( 7 );
- fail( "I should throw an exception" );
- }
- catch ( IllegalArgumentException iae ) {
- //swallow, it's outside the range
- }
- }
-
-
- @Test( expected = IllegalArgumentException.class )
- public void testNoInput() {
- new VersionedMigrationSet<TestVersionImpl>();
- }
-
-
- /**
- * Create the test version impl. Just returns the version provided
- */
- private static final class TestVersionImpl implements VersionedData {
-
- private final int version;
-
-
- private TestVersionImpl( final int version ) {this.version = version;}
-
-
- @Override
- public int getImplementationVersion() {
- return version;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cf32557a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/guice/GraphModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/guice/GraphModule.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/guice/GraphModule.java
index d504e47..91a0b88 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/guice/GraphModule.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/guice/GraphModule.java
@@ -24,12 +24,11 @@ import org.safehaus.guicyfig.GuicyFigModule;
import org.apache.usergrid.persistence.core.consistency.TimeService;
import org.apache.usergrid.persistence.core.consistency.TimeServiceImpl;
import org.apache.usergrid.persistence.core.guice.ProxyImpl;
-import org.apache.usergrid.persistence.core.migration.data.newimpls.DataMigration2;
-import org.apache.usergrid.persistence.core.migration.data.newimpls.MigrationPlugin;
-import org.apache.usergrid.persistence.core.migration.data.newimpls.MigrationRelationship;
-import org.apache.usergrid.persistence.core.migration.data.newimpls.VersionedMigrationSet;
+import org.apache.usergrid.persistence.core.migration.data.DataMigration;
+import org.apache.usergrid.persistence.core.migration.data.MigrationPlugin;
+import org.apache.usergrid.persistence.core.migration.data.MigrationRelationship;
+import org.apache.usergrid.persistence.core.migration.data.VersionedMigrationSet;
import org.apache.usergrid.persistence.core.migration.schema.Migration;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.task.NamedTaskExecutorImpl;
import org.apache.usergrid.persistence.core.task.TaskExecutor;
import org.apache.usergrid.persistence.graph.GraphFig;
@@ -56,6 +55,7 @@ import org.apache.usergrid.persistence.graph.serialization.impl.EdgesObservableI
import org.apache.usergrid.persistence.graph.serialization.impl.GraphManagerFactoryImpl;
import org.apache.usergrid.persistence.graph.serialization.impl.NodeSerializationImpl;
import org.apache.usergrid.persistence.graph.serialization.impl.TargetIdObservableImpl;
+import org.apache.usergrid.persistence.graph.serialization.impl.migration.GraphMigration;
import org.apache.usergrid.persistence.graph.serialization.impl.migration.GraphMigrationPlugin;
import org.apache.usergrid.persistence.graph.serialization.impl.migration.GraphNode;
import org.apache.usergrid.persistence.graph.serialization.impl.shard.EdgeColumnFamilies;
@@ -131,8 +131,8 @@ public abstract class GraphModule extends AbstractModule {
//wire up the edg migration
- Multibinder<DataMigration2<GraphNode>> dataMigrationMultibinder =
- Multibinder.newSetBinder( binder(), new TypeLiteral<DataMigration2<GraphNode>>() {} );
+ Multibinder<DataMigration<GraphNode>> dataMigrationMultibinder =
+ Multibinder.newSetBinder( binder(), new TypeLiteral<DataMigration<GraphNode>>() {}, GraphMigration.class );
dataMigrationMultibinder.addBinding().to( EdgeDataMigrationImpl.class );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cf32557a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/EdgeMetadataSerialization.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/EdgeMetadataSerialization.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/EdgeMetadataSerialization.java
index a843b3c..614b4bf 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/EdgeMetadataSerialization.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/EdgeMetadataSerialization.java
@@ -22,7 +22,7 @@ package org.apache.usergrid.persistence.graph.serialization;
import java.util.Iterator;
-import org.apache.usergrid.persistence.core.migration.data.newimpls.VersionedData;
+import org.apache.usergrid.persistence.core.migration.data.VersionedData;
import org.apache.usergrid.persistence.core.migration.schema.Migration;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.graph.Edge;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cf32557a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeMetadataSerializationProxyImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeMetadataSerializationProxyImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeMetadataSerializationProxyImpl.java
index 262d665..a3d48de 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeMetadataSerializationProxyImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeMetadataSerializationProxyImpl.java
@@ -31,8 +31,8 @@ import org.slf4j.LoggerFactory;
import org.apache.usergrid.persistence.core.astyanax.MultiTennantColumnFamilyDefinition;
import org.apache.usergrid.persistence.core.migration.data.MigrationInfoCache;
-import org.apache.usergrid.persistence.core.migration.data.newimpls.MigrationRelationship;
-import org.apache.usergrid.persistence.core.migration.data.newimpls.VersionedMigrationSet;
+import org.apache.usergrid.persistence.core.migration.data.MigrationRelationship;
+import org.apache.usergrid.persistence.core.migration.data.VersionedMigrationSet;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.graph.Edge;
import org.apache.usergrid.persistence.graph.SearchEdgeType;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cf32557a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/EdgeDataMigrationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/EdgeDataMigrationImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/EdgeDataMigrationImpl.java
index af157a6..2d9b47f 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/EdgeDataMigrationImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/EdgeDataMigrationImpl.java
@@ -23,11 +23,11 @@ import com.google.inject.Inject;
import com.netflix.astyanax.Keyspace;
import com.netflix.astyanax.MutationBatch;
import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
-import org.apache.usergrid.persistence.core.migration.data.newimpls.DataMigration2;
-import org.apache.usergrid.persistence.core.migration.data.newimpls.MigrationDataProvider;
-import org.apache.usergrid.persistence.core.migration.data.newimpls.MigrationRelationship;
-import org.apache.usergrid.persistence.core.migration.data.newimpls.ProgressObserver;
-import org.apache.usergrid.persistence.core.migration.data.newimpls.VersionedMigrationSet;
+import org.apache.usergrid.persistence.core.migration.data.DataMigration;
+import org.apache.usergrid.persistence.core.migration.data.MigrationDataProvider;
+import org.apache.usergrid.persistence.core.migration.data.MigrationRelationship;
+import org.apache.usergrid.persistence.core.migration.data.ProgressObserver;
+import org.apache.usergrid.persistence.core.migration.data.VersionedMigrationSet;
import org.apache.usergrid.persistence.graph.Edge;
import org.apache.usergrid.persistence.graph.GraphManager;
import org.apache.usergrid.persistence.graph.GraphManagerFactory;
@@ -50,7 +50,7 @@ import java.util.concurrent.atomic.AtomicLong;
* The migration data provider assumes that is will visit every node in the graph
* all edges from these source node will then be re-indexed.
*/
-public class EdgeDataMigrationImpl implements DataMigration2<GraphNode> {
+public class EdgeDataMigrationImpl implements DataMigration<GraphNode> {
private static final Logger logger = LoggerFactory.getLogger(EdgeDataMigrationImpl.class);
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cf32557a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/GraphMigration.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/GraphMigration.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/GraphMigration.java
new file mode 100644
index 0000000..245506a
--- /dev/null
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/GraphMigration.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence.graph.serialization.impl.migration;/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+import java.lang.annotation.Retention;
+import java.lang.annotation.Target;
+
+import com.google.inject.BindingAnnotation;
+
+import static java.lang.annotation.ElementType.FIELD;
+import static java.lang.annotation.ElementType.METHOD;
+import static java.lang.annotation.ElementType.PARAMETER;
+import static java.lang.annotation.RetentionPolicy.RUNTIME;
+
+
+@BindingAnnotation
+@Target({ FIELD, PARAMETER, METHOD }) @Retention(RUNTIME)
+public @interface GraphMigration {}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cf32557a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/GraphMigrationPlugin.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/GraphMigrationPlugin.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/GraphMigrationPlugin.java
index c989822..8da102d 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/GraphMigrationPlugin.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/GraphMigrationPlugin.java
@@ -27,9 +27,9 @@ package org.apache.usergrid.persistence.graph.serialization.impl.migration;
import java.util.Set;
import org.apache.usergrid.persistence.core.migration.data.MigrationInfoSerialization;
-import org.apache.usergrid.persistence.core.migration.data.newimpls.AbstractMigrationPlugin;
-import org.apache.usergrid.persistence.core.migration.data.newimpls.DataMigration2;
-import org.apache.usergrid.persistence.core.migration.data.newimpls.MigrationDataProvider;
+import org.apache.usergrid.persistence.core.migration.data.AbstractMigrationPlugin;
+import org.apache.usergrid.persistence.core.migration.data.DataMigration;
+import org.apache.usergrid.persistence.core.migration.data.MigrationDataProvider;
import com.google.inject.Inject;
import com.google.inject.Singleton;
@@ -46,7 +46,8 @@ public class GraphMigrationPlugin extends AbstractMigrationPlugin<GraphNode> {
@Inject
- public GraphMigrationPlugin( final Set<DataMigration2<GraphNode>> entityDataMigrations,
+ public GraphMigrationPlugin(@GraphMigration
+ final Set<DataMigration<GraphNode>> entityDataMigrations,
final MigrationDataProvider<GraphNode> entityIdScopeDataMigrationProvider,
final MigrationInfoSerialization migrationInfoSerialization ) {
super( entityDataMigrations, entityIdScopeDataMigrationProvider, migrationInfoSerialization );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cf32557a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/guice/TestGraphModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/guice/TestGraphModule.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/guice/TestGraphModule.java
index 3201a92..42411d3 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/guice/TestGraphModule.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/guice/TestGraphModule.java
@@ -21,9 +21,8 @@ package org.apache.usergrid.persistence.graph.guice;
import org.apache.usergrid.persistence.core.guice.CommonModule;
import org.apache.usergrid.persistence.core.guice.TestModule;
-import org.apache.usergrid.persistence.core.migration.data.newimpls.MigrationDataProvider;
-import org.apache.usergrid.persistence.core.migration.data.newimpls.TestMigrationDataProvider;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.core.migration.data.MigrationDataProvider;
+import org.apache.usergrid.persistence.core.migration.data.TestMigrationDataProvider;
import org.apache.usergrid.persistence.graph.serialization.impl.migration.GraphNode;
import com.google.inject.TypeLiteral;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cf32557a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/EdgeDataMigrationImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/EdgeDataMigrationImplTest.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/EdgeDataMigrationImplTest.java
index 09259f1..a269c15 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/EdgeDataMigrationImplTest.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/EdgeDataMigrationImplTest.java
@@ -29,7 +29,7 @@ import org.junit.runner.RunWith;
import org.apache.usergrid.persistence.core.guice.DataMigrationResetRule;
import org.apache.usergrid.persistence.core.migration.data.DataMigrationManager;
import org.apache.usergrid.persistence.core.migration.data.TestProgressObserver;
-import org.apache.usergrid.persistence.core.migration.data.newimpls.MigrationDataProvider;
+import org.apache.usergrid.persistence.core.migration.data.MigrationDataProvider;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
import org.apache.usergrid.persistence.core.test.ITRunner;
@@ -142,7 +142,7 @@ public class EdgeDataMigrationImplTest implements DataMigrationResetRule.DataMig
//perform the migration
assertEquals( "Correct version returned", returned, GraphDataVersions.META_SHARDING.getVersion() );
- assertFalse( "Progress observer should not have failed", progressObserver.getFailed() );
+ assertFalse( "Progress observer should not have failed", progressObserver.isFailed() );
assertTrue( "Progress observer should have update messages", progressObserver.getUpdates().size() > 0 );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cf32557a/stack/rest/src/main/java/org/apache/usergrid/rest/MigrateResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/MigrateResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/MigrateResource.java
index 84b1085..9ab5dc3 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/MigrateResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/MigrateResource.java
@@ -77,13 +77,13 @@ public class MigrateResource extends AbstractContextResource {
@Override
public void run() {
- logger.info( "Rebuilding all indexes" );
+ logger.info( "Migrating Data " );
try {
emf.migrateData();
}
catch ( Exception e ) {
- logger.error( "Unable to rebuild indexes", e );
+ logger.error( "Unable to migrate data", e );
}
}
};
[29/50] [abbrv] incubator-usergrid git commit: Added more tests
Posted by sf...@apache.org.
Added more tests
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/0c3b4395
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/0c3b4395
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/0c3b4395
Branch: refs/heads/USERGRID-480
Commit: 0c3b4395ef76b22829871e54e0432b2e79158d7c
Parents: 74a6c35
Author: Todd Nine <tn...@apigee.com>
Authored: Sat Mar 7 10:49:25 2015 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Sat Mar 7 10:49:25 2015 -0700
----------------------------------------------------------------------
.../data/DataMigrationManagerImpl.java | 2 +-
.../data/DataMigrationManagerImplTest.java | 115 ++++++++++++++++++-
2 files changed, 115 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0c3b4395/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManagerImpl.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManagerImpl.java
index fe9e741..ad87724 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManagerImpl.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManagerImpl.java
@@ -142,7 +142,7 @@ public class DataMigrationManagerImpl implements DataMigrationManager {
final int highestAllowed = plugin.getMaxVersion();
Preconditions.checkArgument( version <= highestAllowed,
- "You cannot set a version higher than the max of " + highestAllowed );
+ "You cannot set a version higher than the max of " + highestAllowed + " for plugin " + pluginName );
Preconditions.checkArgument( version >= 0, "You must specify a version of 0 or greater" );
migrationInfoSerialization.setVersion( pluginName, version );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0c3b4395/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManagerImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManagerImplTest.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManagerImplTest.java
index 453958e..319282c 100644
--- a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManagerImplTest.java
+++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManagerImplTest.java
@@ -32,6 +32,7 @@ import org.apache.usergrid.persistence.core.migration.schema.MigrationException;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.inOrder;
import static org.mockito.Mockito.mock;
@@ -188,10 +189,122 @@ public class DataMigrationManagerImplTest {
migrationManager.migrate();
-
//we want to verify the bootsrap plugin was called first
InOrder inOrderVerification = inOrder( plugin1, plugin2 );
inOrderVerification.verify( plugin2 ).run( any( ProgressObserver.class ) );
inOrderVerification.verify( plugin1 ).run( any( ProgressObserver.class ) );
}
+
+
+ /**
+ * Happy path of version reset
+ */
+ @Test
+ public void testResetToVersion() {
+
+ final String name = "plugin1";
+ final int version = 10;
+
+ //linked hash set is intentional here. For iteration order we can boostrap to come second so we can
+ //verify it was actually run first
+ final Set<MigrationPlugin> plugins = new LinkedHashSet<>();
+
+ MigrationPlugin plugin1 = mock( MigrationPlugin.class );
+ when( plugin1.getPhase() ).thenReturn( PluginPhase.MIGRATE );
+
+
+ when( plugin1.getName() ).thenReturn( name );
+ when( plugin1.getMaxVersion() ).thenReturn( version );
+
+ plugins.add( plugin1 );
+
+
+ final MigrationInfoSerialization migrationInfoSerialization = mock( MigrationInfoSerialization.class );
+
+
+ DataMigrationManagerImpl migrationManager = new DataMigrationManagerImpl( plugins, migrationInfoSerialization );
+
+ migrationManager.resetToVersion( name, 0 );
+
+ verify( migrationInfoSerialization ).setVersion( name, 0 );
+
+
+ migrationManager.resetToVersion( name, version );
+
+ verify( migrationInfoSerialization ).setVersion( name, version );
+ }
+
+
+ /**
+ * Reset of version that is too high or too low
+ */
+ @Test( expected = IllegalArgumentException.class )
+ public void testResetToInvalidVersions() {
+ final String name = "plugin1";
+ final int version = 10;
+
+ //linked hash set is intentional here. For iteration order we can boostrap to come second so we can
+ //verify it was actually run first
+ final Set<MigrationPlugin> plugins = new LinkedHashSet<>();
+
+ MigrationPlugin plugin1 = mock( MigrationPlugin.class );
+ when( plugin1.getPhase() ).thenReturn( PluginPhase.MIGRATE );
+
+
+ when( plugin1.getName() ).thenReturn( name );
+ when( plugin1.getMaxVersion() ).thenReturn( version );
+
+ plugins.add( plugin1 );
+
+
+ final MigrationInfoSerialization migrationInfoSerialization = mock( MigrationInfoSerialization.class );
+
+
+ DataMigrationManagerImpl migrationManager = new DataMigrationManagerImpl( plugins, migrationInfoSerialization );
+
+ migrationManager.resetToVersion( name, version + 1 );
+ }
+
+
+ /**
+ * Reset with no plugin name
+ */
+ @Test( expected = IllegalArgumentException.class )
+ public void testResetInvalidName() {
+ final String name = "plugin1";
+ final int version = 10;
+
+ //linked hash set is intentional here. For iteration order we can boostrap to come second so we can
+ //verify it was actually run first
+ final Set<MigrationPlugin> plugins = new LinkedHashSet<>();
+
+ MigrationPlugin plugin1 = mock( MigrationPlugin.class );
+ when( plugin1.getPhase() ).thenReturn( PluginPhase.MIGRATE );
+
+
+ when( plugin1.getName() ).thenReturn( name );
+ when( plugin1.getMaxVersion() ).thenReturn( version );
+
+ plugins.add( plugin1 );
+
+
+ final MigrationInfoSerialization migrationInfoSerialization = mock( MigrationInfoSerialization.class );
+
+
+ DataMigrationManagerImpl migrationManager = new DataMigrationManagerImpl( plugins, migrationInfoSerialization );
+
+ migrationManager.resetToVersion( name + "foo", version );
+ }
+
+
+ @Test
+ public void testLastStatus() {
+ fail( "writeme" );
+ }
+
+
+ @Test
+ public void testLastStatusNoPlugin() {
+ fail( "writeme" );
+ }
}
[45/50] [abbrv] incubator-usergrid git commit: Optimizes import to
avoid merge errors
Posted by sf...@apache.org.
Optimizes import to avoid merge errors
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/8ea46baf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/8ea46baf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/8ea46baf
Branch: refs/heads/USERGRID-480
Commit: 8ea46baf6cd79b9ceca9cfac2f56b6e463a0ba40
Parents: b35551a
Author: Todd Nine <tn...@apigee.com>
Authored: Thu Mar 19 10:25:31 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Thu Mar 19 10:25:31 2015 -0600
----------------------------------------------------------------------
.../index/impl/EsEntityIndexImpl.java | 99 ++++++++++----------
1 file changed, 48 insertions(+), 51 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8ea46baf/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index e66bd8d..8814583 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -18,55 +18,23 @@
package org.apache.usergrid.persistence.index.impl;
-import com.codahale.metrics.Meter;
-import com.codahale.metrics.Timer;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.util.concurrent.Futures;
-import com.google.common.util.concurrent.ListenableFuture;
-import com.google.inject.Inject;
-import com.google.inject.assistedinject.Assisted;
-import com.yammer.metrics.core.Clock;
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.lang3.ArrayUtils;
-import org.apache.usergrid.persistence.core.future.BetterFuture;
-import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.core.util.Health;
-import org.apache.usergrid.persistence.core.util.ValidationUtils;
-import org.apache.usergrid.persistence.index.*;
-import org.apache.usergrid.persistence.index.exceptions.IndexException;
-import org.apache.usergrid.persistence.index.query.CandidateResult;
-import org.apache.usergrid.persistence.index.query.CandidateResults;
-import org.apache.usergrid.persistence.index.query.Query;
-import org.apache.usergrid.persistence.index.utils.UUIDUtils;
-import org.apache.usergrid.persistence.map.MapManager;
-import org.apache.usergrid.persistence.map.MapManagerFactory;
-import org.apache.usergrid.persistence.map.MapScope;
-import org.apache.usergrid.persistence.map.impl.MapScopeImpl;
-import org.apache.usergrid.persistence.model.entity.Id;
-import org.apache.usergrid.persistence.model.entity.SimpleId;
-import org.apache.usergrid.persistence.model.util.UUIDGenerator;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
import org.elasticsearch.action.ActionFuture;
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.ListenableActionFuture;
-import org.elasticsearch.action.ShardOperationFailedException;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksRequest;
import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksResponse;
-import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse;
-import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
-import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse;
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder;
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse;
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse;
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse;
-import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateResponse;
-import org.elasticsearch.action.deletebyquery.DeleteByQueryResponse;
-import org.elasticsearch.action.deletebyquery.IndexDeleteByQueryResponse;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchScrollRequestBuilder;
@@ -75,11 +43,12 @@ import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.index.query.*;
+import org.elasticsearch.index.query.FilterBuilder;
+import org.elasticsearch.index.query.MatchAllQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.indices.IndexAlreadyExistsException;
import org.elasticsearch.indices.IndexMissingException;
-import org.elasticsearch.indices.InvalidAliasNameException;
-import org.elasticsearch.rest.action.admin.indices.alias.delete.AliasesMissingException;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.sort.FieldSortBuilder;
@@ -87,17 +56,45 @@ import org.elasticsearch.search.sort.SortBuilders;
import org.elasticsearch.search.sort.SortOrder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import rx.Observable;
-import rx.functions.Func1;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.UUID;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.ArrayUtils;
+
+import org.apache.usergrid.persistence.core.future.BetterFuture;
+import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.core.util.Health;
+import org.apache.usergrid.persistence.core.util.ValidationUtils;
+import org.apache.usergrid.persistence.index.AliasedEntityIndex;
+import org.apache.usergrid.persistence.index.EntityIndexBatch;
+import org.apache.usergrid.persistence.index.IndexBufferProducer;
+import org.apache.usergrid.persistence.index.IndexFig;
+import org.apache.usergrid.persistence.index.IndexIdentifier;
+import org.apache.usergrid.persistence.index.IndexOperationMessage;
+import org.apache.usergrid.persistence.index.IndexScope;
+import org.apache.usergrid.persistence.index.SearchTypes;
+import org.apache.usergrid.persistence.index.exceptions.IndexException;
+import org.apache.usergrid.persistence.index.query.CandidateResult;
+import org.apache.usergrid.persistence.index.query.CandidateResults;
+import org.apache.usergrid.persistence.index.query.Query;
+import org.apache.usergrid.persistence.map.MapManager;
+import org.apache.usergrid.persistence.map.MapManagerFactory;
+import org.apache.usergrid.persistence.map.MapScope;
+import org.apache.usergrid.persistence.map.impl.MapScopeImpl;
+import org.apache.usergrid.persistence.model.entity.Id;
+import org.apache.usergrid.persistence.model.entity.SimpleId;
+import org.apache.usergrid.persistence.model.util.UUIDGenerator;
+
+import com.codahale.metrics.Timer;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
+import com.google.inject.Inject;
+import com.google.inject.assistedinject.Assisted;
-import static org.apache.usergrid.persistence.index.impl.IndexingUtils.*;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.BOOLEAN_PREFIX;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.NUMBER_PREFIX;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.SPLITTER;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.STRING_PREFIX;
/**
@@ -283,7 +280,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
// add read alias
aliasesRequestBuilder.addAlias( indexName, alias.getReadAlias());
- logger.info( "Created new read Alias Name [{}] on Index [{}]", alias.getReadAlias(), indexName);
+ logger.info( "Created new read Alias Name [{}] on Index [{}]", alias.getReadAlias(), indexName );
//add write alias
[46/50] [abbrv] incubator-usergrid git commit: merge 408
Posted by sf...@apache.org.
merge 408
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/b5ce0856
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/b5ce0856
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/b5ce0856
Branch: refs/heads/USERGRID-480
Commit: b5ce08566ba8442f0e69d16c0e7714103c0e8d14
Parents: b35551a 679812e
Author: Shawn Feldman <sf...@apache.org>
Authored: Thu Mar 19 10:37:14 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Thu Mar 19 10:37:14 2015 -0600
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 5 +-
.../corepersistence/CpEntityManagerFactory.java | 39 +-
.../usergrid/corepersistence/ManagerCache.java | 3 +-
.../usergrid/persistence/EntityManager.java | 3 +-
.../cassandra/EntityManagerImpl.java | 4 +-
.../org/apache/usergrid/CoreApplication.java | 7 +-
.../org/apache/usergrid/CoreITSetupImpl.java | 1 +
.../PerformanceEntityRebuildIndexTest.java | 80 ++--
.../index/ApplicationEntityIndex.java | 42 +++
.../usergrid/persistence/index/EntityIndex.java | 46 +--
.../persistence/index/EntityIndexFactory.java | 2 +-
.../persistence/index/IndexIdentifier.java | 25 +-
.../usergrid/persistence/index/SearchType.java | 49 +++
.../usergrid/persistence/index/SearchTypes.java | 14 +-
.../persistence/index/guice/IndexModule.java | 5 +-
.../impl/EsApplicationEntityIndexImpl.java | 305 +++++++++++++++
.../index/impl/EsEntityIndexBatchImpl.java | 45 +--
.../index/impl/EsEntityIndexFactoryImpl.java | 27 +-
.../index/impl/EsEntityIndexImpl.java | 375 ++-----------------
.../persistence/index/impl/EsQueryVistor.java | 88 +++--
.../persistence/index/impl/IndexingUtils.java | 136 ++++---
.../index/impl/CorePerformanceIT.java | 20 +-
.../impl/EntityConnectionIndexImplTest.java | 20 +-
.../persistence/index/impl/EntityIndexTest.java | 159 +++-----
24 files changed, 756 insertions(+), 744 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b5ce0856/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index f76b9fc,41bcf62..e70e538
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@@ -15,24 -15,20 +15,25 @@@
*/
package org.apache.usergrid.corepersistence;
-import com.google.common.base.Optional;
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheLoader;
-import com.google.common.cache.LoadingCache;
-import com.google.inject.Guice;
-import com.google.inject.Injector;
-import static java.lang.String.CASE_INSENSITIVE_ORDER;
-import java.util.*;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.UUID;
import java.util.concurrent.atomic.AtomicBoolean;
++import org.apache.usergrid.persistence.index.ApplicationEntityIndex;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.BeansException;
+import org.springframework.context.ApplicationContext;
+import org.springframework.context.ApplicationContextAware;
+
import org.apache.commons.lang.StringUtils;
-import org.apache.usergrid.corepersistence.rx.AllEntitiesInSystemObservable;
import org.apache.usergrid.corepersistence.util.CpNamingUtils;
import org.apache.usergrid.persistence.AbstractEntity;
import org.apache.usergrid.persistence.Entity;
@@@ -118,15 -105,16 +119,17 @@@ public class CpEntityManagerFactory imp
private CassandraService cassandraService;
private CounterUtils counterUtils;
private Injector injector;
++ private final EntityIndex entityIndex;
private final MetricsFactory metricsFactory;
public CpEntityManagerFactory(
-- final CassandraService cassandraService, final CounterUtils counterUtils, final Injector injector) {
++ final CassandraService cassandraService, final CounterUtils counterUtils, final Injector injector, final EntityIndex entityIndex) {
this.cassandraService = cassandraService;
this.counterUtils = counterUtils;
this.injector = injector;
++ this.entityIndex = entityIndex;
this.managerCache = injector.getInstance( ManagerCache.class );
- this.dataMigrationManager = injector.getInstance( DataMigrationManager.class );
this.metricsFactory = injector.getInstance( MetricsFactory.class );
this.orgApplicationCache = new OrgApplicationCacheImpl( this );
@@@ -311,10 -291,9 +316,9 @@@
properties = new TreeMap<String, Object>( CASE_INSENSITIVE_ORDER );
}
properties.put( PROPERTY_NAME, appName );
- EntityManager appEm = getEntityManager( applicationId );
+ EntityManager appEm = getEntityManager( applicationId);
- appEm.create( applicationId, TYPE_APPLICATION, properties );
- appEm.createIndex();
- appEm.create(applicationId, TYPE_APPLICATION, properties );
++ appEm.create(applicationId, TYPE_APPLICATION, properties);
appEm.resetRoles();
appEm.refreshIndex();
@@@ -354,9 -333,9 +358,6 @@@
em.delete(appToDelete);
}
-- // delete the application's index
-- EntityIndex ei = managerCache.getEntityIndex(new ApplicationScopeImpl(new SimpleId(applicationId, TYPE_APPLICATION)));
-- ei.deleteIndex();
em.refreshIndex();
}
@@@ -367,7 -346,7 +368,7 @@@
// remove old delete_appinfos Entity
EntityManager em = getEntityManager(CpNamingUtils.SYSTEM_APP_ID);
Query q = Query.fromQL(String.format("select * where applicationUuid = '%s'", applicationId.toString()));
-- Results results = em.searchCollection( em.getApplicationRef(), "deleted_appinfos", q);
++ Results results = em.searchCollection(em.getApplicationRef(), "deleted_appinfos", q);
Entity appToRestore = results.getEntity();
if ( appToRestore == null ) {
@@@ -413,7 -392,7 +414,7 @@@
@Override
public Map<String, UUID> getApplications() throws Exception {
-- return getApplications( false );
++ return getApplications(false);
}
@@@ -664,9 -644,9 +665,7 @@@
maybeCreateIndexes();
// system app
-- for ( EntityIndex index : getManagementIndexes() ) {
-- index.refresh();
-- }
++ entityIndex.refresh();
}
@@@ -676,13 -656,13 +675,11 @@@
return;
}
-- for ( EntityIndex index : getManagementIndexes() ) {
-- index.initializeIndex();
-- }
++ entityIndex.initializeIndex();
}
-- private List<EntityIndex> getManagementIndexes() {
++ private List<ApplicationEntityIndex> getManagementIndexes() {
return Arrays.asList( managerCache.getEntityIndex(
new ApplicationScopeImpl( new SimpleId( CpNamingUtils.SYSTEM_APP_ID, "application" ) ) ),
@@@ -764,7 -770,7 +760,6 @@@
@Override
public void addIndex(final UUID applicationId,final String indexSuffix,final int shards,final int replicas, final String writeConsistency){
-- EntityIndex entityIndex = managerCache.getEntityIndex(CpNamingUtils.getApplicationScope(applicationId));
entityIndex.addIndex(indexSuffix, shards, replicas,writeConsistency);
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b5ce0856/stack/core/src/main/java/org/apache/usergrid/corepersistence/ManagerCache.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b5ce0856/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
----------------------------------------------------------------------
diff --cc stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
index f1f165d,f1f165d..c58750b
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
@@@ -25,6 -25,6 +25,7 @@@ import java.util.UUID
import java.util.concurrent.TimeUnit;
import org.apache.commons.lang.RandomStringUtils;
++import org.apache.usergrid.persistence.index.ApplicationEntityIndex;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@@ -175,17 -175,17 +176,17 @@@ public class PerformanceEntityRebuildIn
// ----------------- delete the system and application indexes
logger.debug("Deleting app index index");
-- //deleteIndex( CpNamingUtils.SYSTEM_APP_ID );
-- deleteIndex( em.getApplicationId() );
--
-- // ----------------- test that we can read them, should fail
--
-- logger.debug("Reading data, should fail this time ");
-- try {
-- readData( em, "testTypes", entityCount, 0 );
-- fail("should have failed to read data");
--
-- } catch (Exception expected) {}
++// //deleteIndex( CpNamingUtils.SYSTEM_APP_ID );
++// deleteIndex( em.getApplicationId() );
++//
++// // ----------------- test that we can read them, should fail
++//
++// logger.debug("Reading data, should fail this time ");
++// try {
++// readData( em, "testTypes", entityCount, 0 );
++// fail("should have failed to read data");
++//
++// } catch (Exception expected) {}
// ----------------- rebuild index for catherders only
@@@ -313,20 -313,20 +314,20 @@@
logger.debug("Deleting app index and system app index");
-- deleteIndex( em.getApplicationId() );
--
-- // deleting sytem app index will interfere with other concurrently running tests
-- //deleteIndex( CpNamingUtils.SYSTEM_APP_ID );
--
--
-- // ----------------- test that we can read them, should fail
--
-- logger.debug("Reading data, should fail this time ");
-- try {
-- readData( em, "testTypes", entityCount, 3 );
-- fail("should have failed to read data");
--
-- } catch (Exception expected) {}
++// deleteIndex( em.getApplicationId() );
++//
++// // deleting sytem app index will interfere with other concurrently running tests
++// //deleteIndex( CpNamingUtils.SYSTEM_APP_ID );
++//
++//
++// // ----------------- test that we can read them, should fail
++//
++// logger.debug("Reading data, should fail this time ");
++// try {
++// readData( em, "testTypes", entityCount, 3 );
++// fail("should have failed to read data");
++//
++// } catch (Exception expected) {}
// ----------------- rebuild index
@@@ -374,21 -374,21 +375,20 @@@
readData( em, "testTypes", entityCount, 3 );
}
-- /**
-- * Delete index for all applications, just need the one to get started.
-- */
-- private void deleteIndex( UUID appUuid ) {
--
-- Injector injector = SpringResource.getInstance().getBean( Injector.class );
-- EntityIndexFactory eif = injector.getInstance( EntityIndexFactory.class );
--
-- Id appId = new SimpleId( appUuid, "application");
-- ApplicationScope scope = new ApplicationScopeImpl( appId );
-- EntityIndex ei = eif.createEntityIndex(scope);
-- EsEntityIndexImpl eeii = (EsEntityIndexImpl)ei;
--
-- eeii.deleteIndex();
-- }
++// /**
++// * Delete index for all applications, just need the one to get started.
++// */
++// private void deleteIndex( UUID appUuid ) {
++//
++// Injector injector = SpringResource.getInstance().getBean( Injector.class );
++// EntityIndexFactory eif = injector.getInstance( EntityIndexFactory.class );
++//
++// Id appId = new SimpleId( appUuid, "application");
++// ApplicationScope scope = new ApplicationScopeImpl( appId );
++// ApplicationEntityIndex ei = eif.createApplicationEntityIndex(scope);
++// EsEntityIndexImpl eeii = (EsEntityIndexImpl)ei;
++//
++// }
private int readData( EntityManager em,
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b5ce0856/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
index 1b3b115,521a4e7..854c976
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
@@@ -33,9 -33,9 +34,9 @@@ import java.util.concurrent.Future
/**
- * Provides indexing of Entities within a scope.
+ * Provides management operations for single index
*/
-public interface EntityIndex {
+public interface EntityIndex extends CPManager {
/**
* This should ONLY ever be called once on application create.
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b5ce0856/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index e66bd8d,ae5e3c3..d510a8d
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@@ -49,16 -33,9 +33,14 @@@ import org.apache.usergrid.persistence.
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
import org.elasticsearch.action.ActionFuture;
- import org.elasticsearch.action.ActionListener;
- import org.elasticsearch.action.ListenableActionFuture;
- import org.elasticsearch.action.ShardOperationFailedException;
+
import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
+import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksRequest;
+import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksResponse;
+import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse;
+import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
+import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse;
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder;
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse;
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
[41/50] [abbrv] incubator-usergrid git commit: Added test to prove
issue
Posted by sf...@apache.org.
Added test to prove issue
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/143692fe
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/143692fe
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/143692fe
Branch: refs/heads/USERGRID-480
Commit: 143692fe9e8d06e1a006636b713a68591377fc1d
Parents: 407a4f5
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Mar 18 16:07:46 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Mar 18 16:07:46 2015 -0600
----------------------------------------------------------------------
.../corepersistence/StaleIndexCleanupTest.java | 48 ++++++++++++++++++++
1 file changed, 48 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/143692fe/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
index 4bde50e..1becee4 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
@@ -127,6 +127,54 @@ public class StaleIndexCleanupTest extends AbstractCoreIT {
}
+
+ /**
+ * USERGRID-492 test for ordering
+ */
+ @Test
+ public void testUpdateVersionMaxFirst() throws Exception {
+
+ // turn off post processing stuff that cleans up stale entities
+ System.setProperty( EVENTS_DISABLED, "true" );
+
+ final EntityManager em = app.getEntityManager();
+
+ Entity thing = em.create( "thing", new HashMap<String, Object>() {{
+ put( "ordinal", 0 );
+ }} );
+
+ em.refreshIndex();
+
+ assertEquals( 1, queryCollectionCp( "things", "thing", "select *" ).size() );
+
+ em.updateProperties( thing, new HashMap<String, Object>() {{
+ put( "ordinal", 1 );
+ }} );
+ em.refreshIndex();
+
+ UUID newVersion = getCpEntity( thing ).getVersion();
+
+
+ assertEquals( 2, queryCollectionCp( "things", "thing", "select * order by ordinal desc" ).size() );
+
+ //now run enable events and ensure we clean up
+ System.setProperty( EVENTS_DISABLED, "false" );
+
+ final Results results = queryCollectionEm( "things", "select * order by ordinal desc" );
+
+ assertEquals( 1, results.size() );
+ assertEquals(1, results.getEntities().get( 0 ).getProperty( "ordinal" ));
+
+ em.refreshIndex();
+
+ //ensure it's actually gone
+ final CandidateResults candidates = queryCollectionCp( "things", "thing", "select * order by ordinal desc" );
+ assertEquals( 1, candidates.size() );
+
+ assertEquals(newVersion, candidates.get( 0 ).getVersion());
+ }
+
+
/**
* Test that the CpRelationManager cleans up and stale indexes that it finds when
* it is building search results.
[50/50] [abbrv] incubator-usergrid git commit: fix collections it
Posted by sf...@apache.org.
fix collections it
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/735eedb2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/735eedb2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/735eedb2
Branch: refs/heads/USERGRID-480
Commit: 735eedb2c25d6149f58728cbde74ca0dbf7d9283
Parents: 70fe285
Author: Shawn Feldman <sf...@apache.org>
Authored: Thu Mar 19 17:24:32 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Thu Mar 19 17:24:32 2015 -0600
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 7 ----
.../corepersistence/CpEntityManagerFactory.java | 6 +--
.../usergrid/corepersistence/CpSetup.java | 6 ++-
.../usergrid/persistence/EntityManager.java | 5 ---
.../cassandra/EntityManagerImpl.java | 6 ---
.../corepersistence/StaleIndexCleanupTest.java | 39 ++++++++++++++++----
.../usergrid/persistence/CollectionIT.java | 14 ++++---
.../index/impl/EsEntityIndexImpl.java | 4 ++
8 files changed, 53 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/735eedb2/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index a9f00b4..64f0257 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -2893,13 +2893,6 @@ public class CpEntityManager implements EntityManager {
@Override
- public void createIndex() {
- ei.initializeIndex();
- }
-
-
-
- @Override
public void flushManagerCaches() {
managerCache.invalidate();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/735eedb2/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index f3d18d0..134886f 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -160,8 +160,8 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
em.getApplication();
}
- em.createIndex();
- em.refreshIndex();
+ entityIndex.initializeIndex();
+ entityIndex.refresh();
} catch (Exception ex) {
@@ -723,7 +723,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
EntityManager em = getEntityManager( appId );
//explicitly invoke create index, we don't know if it exists or not in ES during a rebuild.
- em.createIndex();
+ entityIndex.initializeIndex();
em.reindex(po);
em.reindex( po );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/735eedb2/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpSetup.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpSetup.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpSetup.java
index d70c16c..a9e6cee 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpSetup.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpSetup.java
@@ -19,6 +19,8 @@ package org.apache.usergrid.corepersistence;
import java.util.UUID;
+import com.google.inject.Binding;
+import org.apache.usergrid.persistence.index.EntityIndex;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -64,6 +66,7 @@ public class CpSetup implements Setup {
private final CassandraService cass;
private final EntityManagerFactory emf;
+ private final EntityIndex entityIndex;
/**
@@ -75,6 +78,7 @@ public class CpSetup implements Setup {
this.emf = emf;
this.cass = cassandraService;
this.injector = injector;
+ this.entityIndex = injector.getInstance(EntityIndex.class);
}
@@ -95,7 +99,7 @@ public class CpSetup implements Setup {
setupStaticKeyspace();
//force the EMF creation of indexes before creating the default applications
- emf.refreshIndex();
+ entityIndex.initializeIndex();
injector.getInstance( DataMigrationManager.class ).migrate();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/735eedb2/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
index 71dcd07..89a5384 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
@@ -690,11 +690,6 @@ public interface EntityManager {
*/
void refreshIndex();
- /**
- * Create the index, should ONLY ever be called the first time an application is created
- */
- void createIndex();
-
public void init( EntityManagerFactory emf, EntityIndex entityIndex, UUID applicationId);
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/735eedb2/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
index 803512b..112b684 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
@@ -2890,12 +2890,6 @@ public class EntityManagerImpl implements EntityManager {
}
- @Override
- public void createIndex() {
- //no op
- }
-
-
@Override
public EntityRef getGroupRoleRef( UUID ownerId, String roleName) throws Exception {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/735eedb2/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
index fdd230c..450f44b 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
@@ -120,7 +120,14 @@ public class StaleIndexCleanupTest extends AbstractCoreIT {
assertTrue( "New version is greater than old",
UUIDComparator.staticCompare( newVersion, oldVersion ) > 0 );
- assertEquals( 2, queryCollectionCp( "things", "thing", "select *" ).size() );
+ CandidateResults results;
+ do{
+ results = queryCollectionCp( "things", "thing", "select *" );
+ if(results.size()!=2){
+ Thread.sleep(200);
+ }
+ }while(results.size()!=2);
+ assertEquals( 2, results.size() );
}
@@ -151,24 +158,42 @@ public class StaleIndexCleanupTest extends AbstractCoreIT {
UUID newVersion = getCpEntity( thing ).getVersion();
+ CandidateResults candidateResults = null;
+
+ do{
+ candidateResults = queryCollectionCp("things", "thing", "select * order by ordinal desc");
+ if(candidateResults.size()!=2){
+ Thread.sleep(200);
+ }
+ }while(candidateResults.size()<2);
- assertEquals(2, queryCollectionCp("things", "thing", "select * order by ordinal desc").size());
+ assertEquals(2, candidateResults.size());
//now run enable events and ensure we clean up
System.setProperty(EVENTS_DISABLED, "false");
- final Results results = queryCollectionEm("things", "select * order by ordinal desc");
-
+ Results results = null;
+ do{
+ results = queryCollectionEm("things", "select * order by ordinal desc");;
+ if(results.size()!=1){
+ Thread.sleep(200);
+ }
+ }while(results.size()<1);
assertEquals( 1, results.size() );
assertEquals(1, results.getEntities().get( 0 ).getProperty( "ordinal" ));
em.refreshIndex();
//ensure it's actually gone
- final CandidateResults candidates = queryCollectionCp( "things", "thing", "select * order by ordinal desc" );
- assertEquals(1, candidates.size());
+ do{
+ candidateResults = queryCollectionCp( "things", "thing", "select * order by ordinal desc" );
+ if(candidateResults.size()!=1){
+ Thread.sleep(200);
+ }
+ }while(candidateResults.size()!=1);
+ assertEquals(1, candidateResults.size());
- assertEquals(newVersion, candidates.get(0).getVersion());
+ assertEquals(newVersion, candidateResults.get(0).getVersion());
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/735eedb2/stack/core/src/test/java/org/apache/usergrid/persistence/CollectionIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/CollectionIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/CollectionIT.java
index 641d3ad..aa75a9a 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/CollectionIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/CollectionIT.java
@@ -1227,11 +1227,15 @@ public class CollectionIT extends AbstractCoreIT {
Query query = new Query();
query.addEqualityFilter( "rootprop1", "simpleprop" );
-
- Results results = em.searchCollection( em.getApplicationRef(), "tests", query );
-
- Entity entity = results.getEntitiesMap().get( saved.getUuid() );
-
+ Entity entity;
+ Results results;
+ do {
+ results = em.searchCollection(em.getApplicationRef(), "tests", query);
+ entity = results.getEntitiesMap().get(saved.getUuid());
+ if (entity == null) {
+ Thread.sleep(200);
+ }
+ }while(entity == null);
assertNotNull( entity );
// query on the nested int value
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/735eedb2/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index e92e133..04fde6a 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -18,6 +18,7 @@
package org.apache.usergrid.persistence.index.impl;
+import com.codahale.metrics.Meter;
import com.codahale.metrics.Timer;
import com.google.common.collect.ImmutableMap;
import com.google.inject.Inject;
@@ -105,6 +106,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
private EsIndexCache aliasCache;
private Timer mappingTimer;
private Timer refreshTimer;
+ private Meter refreshIndexMeter;
// private final Timer indexTimer;
@@ -131,6 +133,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
.getTimer(EsEntityIndexImpl.class, "create.mapping.timer");
this.refreshTimer = metricsFactory
.getTimer(EsEntityIndexImpl.class, "refresh.timer");
+ this.refreshIndexMeter = metricsFactory.getMeter(EsEntityIndexImpl.class,"refresh.meter");
}
@@ -323,6 +326,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
public void refresh() {
+ refreshIndexMeter.mark();
BetterFuture future = indexBatchBufferProducer.put(new IndexOperationMessage());
future.get();
[12/50] [abbrv] incubator-usergrid git commit: Pushed migration tests
down to core tiers. WIP
Posted by sf...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/802dcdec/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/GraphShardVersionMigrationIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/GraphShardVersionMigrationIT.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/GraphShardVersionMigrationIT.java
deleted file mode 100644
index f390723..0000000
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/GraphShardVersionMigrationIT.java
+++ /dev/null
@@ -1,235 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.migration;
-
-
-import java.util.HashSet;
-import java.util.Set;
-
-import org.apache.usergrid.corepersistence.CpSetup;
-import org.apache.usergrid.persistence.collection.CollectionScope;
-import org.apache.usergrid.persistence.core.migration.data.*;
-import org.apache.usergrid.persistence.core.rx.AllEntitiesInSystemObservable;
-import org.apache.usergrid.persistence.core.rx.ApplicationObservable;
-import org.apache.usergrid.persistence.core.scope.ApplicationEntityGroup;
-import org.apache.usergrid.persistence.core.scope.EntityIdScope;
-import org.apache.usergrid.persistence.graph.serialization.impl.EdgeDataMigrationImpl;
-import org.apache.usergrid.persistence.graph.serialization.impl.EdgeMetadataSerializationProxyImpl;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Rule;
-import org.junit.Test;
-
-import org.apache.usergrid.AbstractCoreIT;
-import org.apache.usergrid.cassandra.SpringResource;
-import org.apache.usergrid.corepersistence.EntityWriteHelper;
-import org.apache.usergrid.corepersistence.ManagerCache;
-import org.apache.usergrid.corepersistence.rx.impl.AllEntitiesInSystemObservableImpl;
-import org.apache.usergrid.persistence.EntityManager;
-import org.apache.usergrid.persistence.graph.GraphManager;
-import org.apache.usergrid.persistence.graph.impl.SimpleSearchEdgeType;
-import org.apache.usergrid.persistence.model.entity.Id;
-
-import com.google.common.collect.HashMultimap;
-import com.google.common.collect.Multimap;
-import com.google.inject.Injector;
-
-import net.jcip.annotations.NotThreadSafe;
-
-import rx.functions.Action1;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
-
-@NotThreadSafe
-public class GraphShardVersionMigrationIT extends AbstractCoreIT {
-
- private Injector injector;
- private ApplicationDataMigration graphShardVersionMigration;
- private ManagerCache managerCache;
- private DataMigrationManager dataMigrationManager;
- private MigrationInfoSerialization migrationInfoSerialization;
-
-
- /**
- * Rule to do the resets we need
- */
- @Rule
- public MigrationTestRule migrationTestRule = new MigrationTestRule( app, SpringResource.getInstance().getBean(Injector.class) ,EdgeDataMigrationImpl.class );
- private AllEntitiesInSystemObservable allEntitiesInSystemObservable;
- private ApplicationObservable applicationObservable;
-
-
- @Before
- public void setup() {
-
- injector = SpringResource.getInstance().getBean( Injector.class );
- graphShardVersionMigration = injector.getInstance( EdgeDataMigrationImpl.class );
- managerCache = injector.getInstance( ManagerCache.class );
- dataMigrationManager = injector.getInstance( DataMigrationManager.class );
- migrationInfoSerialization = injector.getInstance( MigrationInfoSerialization.class );
- allEntitiesInSystemObservable = injector.getInstance(AllEntitiesInSystemObservable.class);
- applicationObservable = injector.getInstance(ApplicationObservable.class);
-
- }
-
-
- @Test
- @Ignore("Ignored awaiting fix for USERGRID-268")
- public void testIdMapping() throws Throwable {
-
- assertEquals( "version 2 expected", 2, graphShardVersionMigration.getVersion() );
- assertEquals( "Previous version expected", 1, dataMigrationManager.getCurrentVersion());
-
-
-
- final EntityManager newAppEm = app.getEntityManager();
-
- final String type1 = "type1thing";
- final String type2 = "type2thing";
- final int size = 10;
-
- final Set<Id> type1Identities = EntityWriteHelper.createTypes( newAppEm, type1, size );
- final Set<Id> type2Identities = EntityWriteHelper.createTypes( newAppEm, type2, size );
-
-
- final Set<Id> allEntities = new HashSet<>();
- allEntities.addAll( type1Identities );
- allEntities.addAll( type2Identities );
-
-
- final TestProgressObserver progressObserver = new TestProgressObserver();
-
-
- //used to validate 1.0 types, and 2.0 types
- final Multimap<Id, String> sourceTypes = HashMultimap.create( 10000, 10 );
- final Multimap<Id, String> targetTypes = HashMultimap.create( 10000, 10 );
-
-
- //read everything in previous version format and put it into our types.
-
- allEntitiesInSystemObservable.getAllEntitiesInSystem( 1000)
- .doOnNext( new Action1<ApplicationEntityGroup<CollectionScope>>() {
- @Override
- public void call(
- final ApplicationEntityGroup<CollectionScope> entity ) {
-
- final GraphManager gm =
- managerCache.getGraphManager( entity.applicationScope );
-
- for ( final EntityIdScope<CollectionScope> idScope : entity.entityIds ) {
- /**
- * Get our edge types from the source
- */
- gm.getEdgeTypesFromSource( new SimpleSearchEdgeType(idScope.getId(), null, null ) )
- .doOnNext(new Action1<String>() {
- @Override
- public void call(final String s) {
- sourceTypes.put(idScope.getId(), s);
- }
- }).toBlocking().lastOrDefault( null );
-
-
- /**
- * Get the edge types to the target
- */
- gm.getEdgeTypesToTarget( new SimpleSearchEdgeType( idScope.getId(), null, null ) )
- .doOnNext( new Action1<String>() {
- @Override
- public void call( final String s ) {
- targetTypes.put( idScope.getId(), s );
- }
- } ).toBlocking().lastOrDefault( null );
-
- allEntities.remove( idScope.getId() );
- }
- }
- } ).toBlocking().lastOrDefault( null );
-
-
- //perform the migration
-
- graphShardVersionMigration.migrate(applicationObservable.getAllApplicationScopes(), progressObserver).toBlocking().last();
-
- assertEquals( "Newly saved entities encounterd", 0, allEntities.size() );
- assertFalse( "Progress observer should not have failed", progressObserver.getFailed() );
- assertTrue( "Progress observer should have update messages", progressObserver.getUpdates().size() > 0 );
-
-
- //write the status and version, then invalidate the cache so it appears
- migrationInfoSerialization.setStatusCode( DataMigrationManagerImpl.StatusCode.COMPLETE.status );
- migrationInfoSerialization.setVersion( graphShardVersionMigration.getVersion() );
- dataMigrationManager.invalidate();
-
- assertEquals( "New version saved, and we should get new implementation",
- graphShardVersionMigration.getVersion(), dataMigrationManager.getCurrentVersion() );
-
-
- //now visit all nodes in the system and remove their types from the multi maps, it should be empty at the end
- allEntitiesInSystemObservable.getAllEntitiesInSystem( 1000)
- .doOnNext( new Action1<ApplicationEntityGroup<CollectionScope>>() {
- @Override
- public void call(
- final ApplicationEntityGroup<CollectionScope> entity ) {
-
- final GraphManager gm =
- managerCache.getGraphManager( entity.applicationScope );
-
- for ( final EntityIdScope<CollectionScope> idScope : entity.entityIds ) {
- /**
- * Get our edge types from the source
- */
- gm.getEdgeTypesFromSource(
- new SimpleSearchEdgeType( idScope.getId(), null, null ) )
- .doOnNext( new Action1<String>() {
- @Override
- public void call( final String s ) {
- sourceTypes.remove( idScope.getId(), s );
- }
- } ).toBlocking().lastOrDefault( null );
-
-
- /**
- * Get the edge types to the target
- */
- gm.getEdgeTypesToTarget(
- new SimpleSearchEdgeType( idScope.getId(), null, null ) )
- .doOnNext( new Action1<String>() {
- @Override
- public void call( final String s ) {
- targetTypes.remove( idScope.getId(), s );
- }
- } ).toBlocking().lastOrDefault( null );
- }
- }
- }
-
-
- ).toBlocking().lastOrDefault( null );
-
-
- assertEquals( "All source types migrated", 0, sourceTypes.size() );
-
-
- assertEquals( "All target types migrated", 0, targetTypes.size() );
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/802dcdec/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/MigrationTestRule.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/MigrationTestRule.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/MigrationTestRule.java
index 3071b72..67f89ab 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/MigrationTestRule.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/MigrationTestRule.java
@@ -25,8 +25,9 @@ import org.junit.runner.Description;
import org.junit.runners.model.Statement;
import org.apache.usergrid.CoreApplication;
-import org.apache.usergrid.persistence.core.migration.data.DataMigration;
import org.apache.usergrid.persistence.core.migration.data.DataMigrationManager;
+import org.apache.usergrid.persistence.core.migration.data.newimpls.DataMigration2;
+import org.apache.usergrid.persistence.core.migration.data.newimpls.MigrationPlugin;
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
import com.google.inject.Injector;
@@ -45,7 +46,7 @@ public class MigrationTestRule extends ExternalResource {
protected final CoreApplication core;
protected final DataMigrationManager dataMigrationManager;
- protected final DataMigration dataMigration;
+ protected final DataMigration2 dataMigration;
protected int currentVersion;
@@ -55,10 +56,8 @@ public class MigrationTestRule extends ExternalResource {
*
* @param core the CoreApplication rule used in this test
* @param injector The injector used in this test
- * @param dataMigrationClass The data migration class that is under test
*/
- public MigrationTestRule( final CoreApplication core, final Injector injector,
- final Class<? extends DataMigration> dataMigrationClass ) {
+ public MigrationTestRule( final CoreApplication core, final Injector injector ) {
this.core = core;
this.dataMigrationManager = injector.getInstance( DataMigrationManager.class );
this.dataMigration = injector.getInstance( dataMigrationClass );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/802dcdec/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/TestProgressObserver.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/TestProgressObserver.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/TestProgressObserver.java
deleted file mode 100644
index c7b69a1..0000000
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/TestProgressObserver.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.migration;
-
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.usergrid.persistence.core.migration.data.DataMigration;
-
-
-public class TestProgressObserver implements DataMigration.ProgressObserver {
-
- private boolean failed = false;
-
-
- private List<String> updates = new ArrayList<>( 100 );
-
-
- @Override
- public void failed( final int migrationVersion, final String reason ) {
- failed = true;
- }
-
-
- @Override
- public void failed( final int migrationVersion, final String reason, final Throwable throwable ) {
- failed = true;
- }
-
-
- @Override
- public void update( final int migrationVersion, final String message ) {
- updates.add( message );
- }
-
-
- /**
- * Get if we failed
- * @return
- */
- public boolean getFailed() {
- return failed;
- }
-
-
- /**
- * Get update messages
- * @return
- */
- public List<String> getUpdates() {
- return updates;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/802dcdec/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/AllEntitiesInSystemObservableIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/AllEntitiesInSystemObservableIT.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/AllEntitiesInSystemObservableIT.java
index ca14bd0..7a6ec22 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/AllEntitiesInSystemObservableIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/AllEntitiesInSystemObservableIT.java
@@ -23,26 +23,20 @@ package org.apache.usergrid.corepersistence.rx;
import java.util.HashSet;
import java.util.Set;
-import org.apache.usergrid.corepersistence.rx.impl.AllEntitiesInSystemObservableImpl;
-import org.apache.usergrid.persistence.collection.CollectionScope;
-import org.apache.usergrid.persistence.core.rx.AllEntitiesInSystemObservable;
-import org.apache.usergrid.persistence.core.rx.ApplicationObservable;
-import org.apache.usergrid.persistence.core.scope.ApplicationEntityGroup;
-import org.apache.usergrid.persistence.core.scope.EntityIdScope;
-import org.apache.usergrid.persistence.graph.serialization.TargetIdObservable;
import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.apache.usergrid.AbstractCoreIT;
import org.apache.usergrid.cassandra.SpringResource;
import org.apache.usergrid.corepersistence.EntityWriteHelper;
import org.apache.usergrid.corepersistence.ManagerCache;
+import org.apache.usergrid.corepersistence.rx.impl.AllEntitiesInSystemImpl;
import org.apache.usergrid.corepersistence.util.CpNamingUtils;
import org.apache.usergrid.persistence.EntityManager;
import org.apache.usergrid.persistence.SimpleEntityRef;
+import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.graph.GraphManager;
+import org.apache.usergrid.persistence.graph.serialization.TargetIdObservable;
import org.apache.usergrid.persistence.model.entity.Id;
import com.google.inject.Injector;
@@ -59,12 +53,10 @@ import static org.junit.Assert.assertTrue;
*/
public class AllEntitiesInSystemObservableIT extends AbstractCoreIT {
- private final Logger logger = LoggerFactory.getLogger( AllEntitiesInSystemObservableIT.class );
-
@Test
public void testEntities() throws Exception {
Injector injector = SpringResource.getInstance().getBean(Injector.class);
- AllEntitiesInSystemObservable allEntitiesInSystemObservableImpl =injector.getInstance(AllEntitiesInSystemObservable.class);
+ AllEntitiesInSystemImpl allEntitiesInSystemObservableImpl =injector.getInstance(AllEntitiesInSystemImpl.class);
TargetIdObservable targetIdObservable = injector.getInstance(TargetIdObservable.class);
final EntityManager em = app.getEntityManager();
@@ -99,34 +91,25 @@ public class AllEntitiesInSystemObservableIT extends AbstractCoreIT {
final ApplicationScope scope = CpNamingUtils.getApplicationScope( app.getId() );
- final Id applicationId = scope.getApplication();
final GraphManager gm = managerCache.getGraphManager( scope );
- allEntitiesInSystemObservableImpl.getAllEntitiesInSystem( 1000).doOnNext( new Action1<ApplicationEntityGroup<CollectionScope>>() {
+ allEntitiesInSystemObservableImpl.getData().doOnNext( new Action1<EntityIdScope>() {
@Override
- public void call( final ApplicationEntityGroup<CollectionScope> entity ) {
-
- assertNotNull(entity);
- assertNotNull(entity.applicationScope);
- assertNotNull(entity.entityIds);
-
- //not from our test, don't check it
- if(!applicationId.equals( entity.applicationScope.getApplication() )){
- return;
- }
-
- for(EntityIdScope<CollectionScope> idScope : entity.entityIds) {
+ public void call( final EntityIdScope entityIdScope ) {
+ assertNotNull(entityIdScope);
+ assertNotNull(entityIdScope.getCollectionScope());
+ assertNotNull(entityIdScope.getId());
//we should only emit each node once
- if ( idScope.getId().getType().equals( type1 ) ) {
- assertTrue( "Element should be present on removal", type1Identities.remove(idScope.getId() ) );
+ if ( entityIdScope.getId().getType().equals( type1 ) ) {
+ assertTrue( "Element should be present on removal", type1Identities.remove(entityIdScope.getId() ) );
}
- else if ( idScope.getId().getType().equals( type2 ) ) {
- assertTrue( "Element should be present on removal", type2Identities.remove(idScope.getId() ) );
+ else if ( entityIdScope.getId().getType().equals( type2 ) ) {
+ assertTrue( "Element should be present on removal", type2Identities.remove(entityIdScope.getId() ) );
}
- }
+
}
} ).toBlocking().lastOrDefault( null );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/802dcdec/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/ApplicationObservableTestIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/ApplicationObservableTestIT.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/ApplicationObservableTestIT.java
index 081bbed..649f518 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/ApplicationObservableTestIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/ApplicationObservableTestIT.java
@@ -24,15 +24,15 @@ import java.util.HashSet;
import java.util.Set;
import java.util.UUID;
-import org.apache.usergrid.persistence.core.rx.ApplicationObservable;
import org.junit.Test;
import org.apache.usergrid.AbstractCoreIT;
import org.apache.usergrid.cassandra.SpringResource;
+import org.apache.usergrid.corepersistence.AllApplicationsObservable;
import org.apache.usergrid.corepersistence.ManagerCache;
import org.apache.usergrid.corepersistence.util.CpNamingUtils;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.entities.Application;
-import org.apache.usergrid.persistence.model.entity.Id;
import com.google.inject.Injector;
@@ -52,7 +52,7 @@ public class ApplicationObservableTestIT extends AbstractCoreIT {
final Application createdApplication = app.getEntityManager().getApplication();
- ApplicationObservable applicationObservable =SpringResource.getInstance().getBean(Injector.class).getInstance(ApplicationObservable.class);
+ AllApplicationsObservable applicationObservable =SpringResource.getInstance().getBean(Injector.class).getInstance(AllApplicationsObservable.class);
//now our get all apps we expect. There may be more, but we don't care about those.
final Set<UUID> applicationIds = new HashSet<UUID>() {{
@@ -67,13 +67,13 @@ public class ApplicationObservableTestIT extends AbstractCoreIT {
//clean up our wiring
ManagerCache managerCache = SpringResource.getInstance().getBean( Injector.class ).getInstance( ManagerCache.class );
- Observable<Id> appObservable = applicationObservable.getAllApplicationIds();
+ Observable<ApplicationScope> appObservable = applicationObservable.getAllApplications();
- appObservable.doOnNext( new Action1<Id>() {
+ appObservable.doOnNext( new Action1<ApplicationScope>() {
@Override
- public void call( final Id id ) {
- applicationIds.remove( id.getUuid() );
- assertEquals("Correct application type expected" , Application.ENTITY_TYPE, id.getType() );
+ public void call( final ApplicationScope id ) {
+ applicationIds.remove( id.getApplication().getUuid() );
+ assertEquals("Correct application type expected" , Application.ENTITY_TYPE, id.getApplication().getType() );
}
} ).toBlocking().lastOrDefault( null );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/802dcdec/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/MvccLogEntry.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/MvccLogEntry.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/MvccLogEntry.java
index e518298..6d32d73 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/MvccLogEntry.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/MvccLogEntry.java
@@ -65,6 +65,7 @@ public interface MvccLogEntry {
/**
* The logentry being written represents a partial entity
*/
+ @Deprecated//removed in v3
PARTIAL(1),
/**
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/802dcdec/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/guice/CollectionModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/guice/CollectionModule.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/guice/CollectionModule.java
index 7d78177..d5478d4 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/guice/CollectionModule.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/guice/CollectionModule.java
@@ -89,28 +89,15 @@ public abstract class CollectionModule extends AbstractModule {
configureMigrationProvider();
}
-
- @Provides
- @Singleton
- @Inject
- @Write
- public WriteStart write (final MvccLogEntrySerializationStrategy logStrategy) {
- final WriteStart writeStart = new WriteStart( logStrategy, MvccEntity.Status.COMPLETE);
-
- return writeStart;
- }
-
-
- //TODO USERGRID-405, remove this, it's no longer supported
- @Provides
- @Singleton
- @Inject
- @WriteUpdate
- public WriteStart writeUpdate (final MvccLogEntrySerializationStrategy logStrategy) {
- final WriteStart writeStart = new WriteStart( logStrategy, MvccEntity.Status.PARTIAL );
-
- return writeStart;
- }
+//
+// @Provides
+// @Singleton
+// @Inject
+// public WriteStart write (final MvccLogEntrySerializationStrategy logStrategy) {
+// final WriteStart writeStart = new WriteStart( logStrategy, MvccEntity.Status.COMPLETE);
+//
+// return writeStart;
+// }
@Inject
@Singleton
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/802dcdec/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/guice/Write.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/guice/Write.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/guice/Write.java
deleted file mode 100644
index ad752af..0000000
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/guice/Write.java
+++ /dev/null
@@ -1,17 +0,0 @@
-package org.apache.usergrid.persistence.collection.guice;
-
-
-import java.lang.annotation.Retention;
-import java.lang.annotation.Target;
-
-import com.google.inject.BindingAnnotation;
-
-import static java.lang.annotation.ElementType.FIELD;
-import static java.lang.annotation.ElementType.METHOD;
-import static java.lang.annotation.ElementType.PARAMETER;
-import static java.lang.annotation.RetentionPolicy.RUNTIME;
-
-@BindingAnnotation
-@Target( { FIELD, PARAMETER, METHOD } )
-@Retention( RUNTIME )
-public @interface Write {}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/802dcdec/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/guice/WriteUpdate.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/guice/WriteUpdate.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/guice/WriteUpdate.java
deleted file mode 100644
index 0ba3991..0000000
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/guice/WriteUpdate.java
+++ /dev/null
@@ -1,17 +0,0 @@
-package org.apache.usergrid.persistence.collection.guice;
-
-
-import java.lang.annotation.Retention;
-import java.lang.annotation.Target;
-
-import com.google.inject.BindingAnnotation;
-
-import static java.lang.annotation.ElementType.FIELD;
-import static java.lang.annotation.ElementType.METHOD;
-import static java.lang.annotation.ElementType.PARAMETER;
-import static java.lang.annotation.RetentionPolicy.RUNTIME;
-
-@BindingAnnotation
-@Target( { FIELD, PARAMETER, METHOD } )
-@Retention( RUNTIME )
-public @interface WriteUpdate {}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/802dcdec/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerFactoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerFactoryImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerFactoryImpl.java
index fa36f42..10d85f8 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerFactoryImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerFactoryImpl.java
@@ -32,8 +32,6 @@ import org.apache.usergrid.persistence.collection.EntityVersionCreatedFactory;
import org.apache.usergrid.persistence.collection.cache.CachedEntityCollectionManager;
import org.apache.usergrid.persistence.collection.cache.EntityCacheFig;
import org.apache.usergrid.persistence.collection.guice.CollectionTaskExecutor;
-import org.apache.usergrid.persistence.collection.guice.Write;
-import org.apache.usergrid.persistence.collection.guice.WriteUpdate;
import org.apache.usergrid.persistence.collection.mvcc.MvccLogEntrySerializationStrategy;
import org.apache.usergrid.persistence.collection.mvcc.stage.delete.MarkCommit;
import org.apache.usergrid.persistence.collection.mvcc.stage.delete.MarkStart;
@@ -66,7 +64,6 @@ public class EntityCollectionManagerFactoryImpl implements EntityCollectionManag
private final WriteStart writeStart;
- private final WriteStart writeUpdate;
private final WriteUniqueVerify writeVerifyUnique;
private final WriteOptimisticVerify writeOptimisticVerify;
private final WriteCommit writeCommit;
@@ -91,7 +88,7 @@ public class EntityCollectionManagerFactoryImpl implements EntityCollectionManag
//create the target EM that will perform logic
final EntityCollectionManager target = new EntityCollectionManagerImpl(
- writeStart, writeUpdate, writeVerifyUnique,
+ writeStart, writeVerifyUnique,
writeOptimisticVerify, writeCommit, rollback, markStart, markCommit,
entitySerializationStrategy, uniqueValueSerializationStrategy,
mvccLogEntrySerializationStrategy, keyspace, serializationFig,entityVersionCleanupFactory,
@@ -107,8 +104,7 @@ public class EntityCollectionManagerFactoryImpl implements EntityCollectionManag
@Inject
- public EntityCollectionManagerFactoryImpl( @Write final WriteStart writeStart,
- @WriteUpdate final WriteStart writeUpdate,
+ public EntityCollectionManagerFactoryImpl( final WriteStart writeStart,
final WriteUniqueVerify writeVerifyUnique,
final WriteOptimisticVerify writeOptimisticVerify,
final WriteCommit writeCommit, final RollbackAction rollback,
@@ -125,7 +121,6 @@ public class EntityCollectionManagerFactoryImpl implements EntityCollectionManag
final SerializationFig serializationFig) {
this.writeStart = writeStart;
- this.writeUpdate = writeUpdate;
this.writeVerifyUnique = writeVerifyUnique;
this.writeOptimisticVerify = writeOptimisticVerify;
this.writeCommit = writeCommit;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/802dcdec/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
index e483cc1..f496cac 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
@@ -28,12 +28,13 @@ import org.slf4j.LoggerFactory;
import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.collection.EntityCollectionManager;
+import org.apache.usergrid.persistence.collection.EntityDeletedFactory;
import org.apache.usergrid.persistence.collection.EntitySet;
+import org.apache.usergrid.persistence.collection.EntityVersionCleanupFactory;
+import org.apache.usergrid.persistence.collection.EntityVersionCreatedFactory;
import org.apache.usergrid.persistence.collection.MvccEntity;
import org.apache.usergrid.persistence.collection.VersionSet;
-import org.apache.usergrid.persistence.collection.guice.Write;
-import org.apache.usergrid.persistence.collection.guice.WriteUpdate;
-import org.apache.usergrid.persistence.collection.serialization.MvccEntitySerializationStrategy;
+import org.apache.usergrid.persistence.collection.guice.CollectionTaskExecutor;
import org.apache.usergrid.persistence.collection.mvcc.MvccLogEntrySerializationStrategy;
import org.apache.usergrid.persistence.collection.mvcc.entity.MvccValidationUtils;
import org.apache.usergrid.persistence.collection.mvcc.stage.CollectionIoEvent;
@@ -44,11 +45,14 @@ import org.apache.usergrid.persistence.collection.mvcc.stage.write.WriteCommit;
import org.apache.usergrid.persistence.collection.mvcc.stage.write.WriteOptimisticVerify;
import org.apache.usergrid.persistence.collection.mvcc.stage.write.WriteStart;
import org.apache.usergrid.persistence.collection.mvcc.stage.write.WriteUniqueVerify;
+import org.apache.usergrid.persistence.collection.serialization.MvccEntitySerializationStrategy;
import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
import org.apache.usergrid.persistence.collection.serialization.UniqueValue;
import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
import org.apache.usergrid.persistence.collection.serialization.UniqueValueSet;
import org.apache.usergrid.persistence.core.guice.ProxyImpl;
+import org.apache.usergrid.persistence.core.task.Task;
+import org.apache.usergrid.persistence.core.task.TaskExecutor;
import org.apache.usergrid.persistence.core.util.Health;
import org.apache.usergrid.persistence.core.util.ValidationUtils;
import org.apache.usergrid.persistence.model.entity.Entity;
@@ -65,12 +69,6 @@ import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
import com.netflix.astyanax.model.ColumnFamily;
import com.netflix.astyanax.model.CqlResult;
import com.netflix.astyanax.serializers.StringSerializer;
-import org.apache.usergrid.persistence.collection.EntityDeletedFactory;
-import org.apache.usergrid.persistence.collection.EntityVersionCleanupFactory;
-import org.apache.usergrid.persistence.collection.EntityVersionCreatedFactory;
-import org.apache.usergrid.persistence.collection.guice.CollectionTaskExecutor;
-import org.apache.usergrid.persistence.core.task.Task;
-import org.apache.usergrid.persistence.core.task.TaskExecutor;
import rx.Observable;
import rx.Subscriber;
@@ -92,7 +90,6 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
//start stages
private final WriteStart writeStart;
- private final WriteStart writeUpdate;
private final WriteUniqueVerify writeVerifyUnique;
private final WriteOptimisticVerify writeOptimisticVerify;
private final WriteCommit writeCommit;
@@ -117,8 +114,7 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
@Inject
public EntityCollectionManagerImpl(
- @Write final WriteStart writeStart,
- @WriteUpdate final WriteStart writeUpdate,
+ final WriteStart writeStart,
final WriteUniqueVerify writeVerifyUnique,
final WriteOptimisticVerify writeOptimisticVerify,
final WriteCommit writeCommit,
@@ -142,7 +138,6 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
MvccValidationUtils.validateCollectionScope( collectionScope );
this.writeStart = writeStart;
- this.writeUpdate = writeUpdate;
this.writeVerifyUnique = writeVerifyUnique;
this.writeOptimisticVerify = writeOptimisticVerify;
this.writeCommit = writeCommit;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/802dcdec/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteStart.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteStart.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteStart.java
index 8cd21e1..92dc69d 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteStart.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteStart.java
@@ -38,7 +38,6 @@ public class WriteStart implements Func1<CollectionIoEvent<Entity>, CollectionIo
private final MvccLogEntrySerializationStrategy logStrategy;
- MvccEntity.Status status;
/**
@@ -46,9 +45,8 @@ public class WriteStart implements Func1<CollectionIoEvent<Entity>, CollectionIo
*/
@Inject
- public WriteStart ( final MvccLogEntrySerializationStrategy logStrategy, MvccEntity.Status status) {
+ public WriteStart ( final MvccLogEntrySerializationStrategy logStrategy) {
this.logStrategy = logStrategy;
- this.status = status;
}
@@ -69,7 +67,7 @@ public class WriteStart implements Func1<CollectionIoEvent<Entity>, CollectionIo
MutationBatch write = logStrategy.write( collectionScope, startEntry );
- final MvccEntityImpl nextStage = new MvccEntityImpl( entityId, newVersion, status, entity );
+ final MvccEntityImpl nextStage = new MvccEntityImpl( entityId, newVersion, MvccEntity.Status.COMPLETE, entity );
if(ioEvent.getEvent().hasVersion()) {
try {
write.execute();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/802dcdec/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/MvccEntityDataMigrationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/MvccEntityDataMigrationImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/MvccEntityDataMigrationImpl.java
index 108a4d8..7580a26 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/MvccEntityDataMigrationImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/MvccEntityDataMigrationImpl.java
@@ -334,56 +334,6 @@ public class MvccEntityDataMigrationImpl implements DataMigration2<EntityIdScope
.toBlocking()
.last();
}
- } ).doOnNext(
- new Action1<List<GroupedObservable<Id, EntityToSaveMessage>>>() {
- @Override
- public void call(
- final List<GroupedObservable<Id, EntityToSaveMessage>> groupedObservables ) {
-
- for ( final GroupedObservable<Id, EntityToSaveMessage> group : groupedObservables ) {
-
- //get the highest
- // entity and run a
- // cleanup task on it
- final EntityToSaveMessage
- maxEntity =
- group.toBlocking()
- .last();
-
- final EntityVersionCleanupTask
- task =
- entityVersionCleanupFactory
- .getTask(
- maxEntity.scope,
- maxEntity.entity
- .getId(),
- maxEntity.entity
- .getVersion() );
-
- /**
- * just run the
- * call in this
- * process, we're
- * already
- * doing parallel
- * this forces a
- * repair of the
- * unique properties,
- and will bring us
- to a consistent
- state after the
- */
-
- try {
- task.call();
- }
- catch ( Exception e ) {
- throw new RuntimeException(
- "Unable to run cleanup task",
- e );
- }
- }
- }
} ).
reduce( 0l,
new Func2<Long, List<GroupedObservable<Id, EntityToSaveMessage>>, Long>() {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/802dcdec/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteStartTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteStartTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteStartTest.java
index a683d23..c00c82a 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteStartTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteStartTest.java
@@ -68,7 +68,7 @@ public class WriteStartTest extends AbstractEntityStageTest {
final Entity entity = TestEntityGenerator.generateEntity();
//run the stage
- WriteStart newStage = new WriteStart( logStrategy, MvccEntity.Status.COMPLETE);
+ WriteStart newStage = new WriteStart( logStrategy);
//verify the observable is correct
@@ -113,7 +113,7 @@ public class WriteStartTest extends AbstractEntityStageTest {
//set up the mock to return the entity from the start phase
final Entity entity = TestEntityGenerator.generateEntity(new SimpleId(UUID.randomUUID(),"test"),null);
//run the stage
- WriteStart newStage = new WriteStart( logStrategy, MvccEntity.Status.COMPLETE);
+ WriteStart newStage = new WriteStart( logStrategy );
//verify the observable is correct
CollectionIoEvent<MvccEntity> result = newStage.call( new CollectionIoEvent<Entity>( context, entity ) );
@@ -140,7 +140,7 @@ public class WriteStartTest extends AbstractEntityStageTest {
protected void validateStage( final CollectionIoEvent<Entity> event ) {
final MvccLogEntrySerializationStrategy logStrategy = mock( MvccLogEntrySerializationStrategy.class );
- new WriteStart( logStrategy, MvccEntity.Status.COMPLETE ).call( event );
+ new WriteStart( logStrategy ).call( event );
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/802dcdec/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/MvccEntityDataMigrationV1ToV3ImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/MvccEntityDataMigrationV1ToV3ImplTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/MvccEntityDataMigrationV1ToV3ImplTest.java
new file mode 100644
index 0000000..dde67cb
--- /dev/null
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/MvccEntityDataMigrationV1ToV3ImplTest.java
@@ -0,0 +1,179 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence.collection.serialization.impl.migration;
+
+
+import java.util.UUID;
+
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+import org.apache.usergrid.persistence.collection.CollectionScope;
+import org.apache.usergrid.persistence.collection.MvccEntity;
+import org.apache.usergrid.persistence.collection.guice.TestCollectionModule;
+import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl;
+import org.apache.usergrid.persistence.collection.mvcc.entity.impl.MvccEntityImpl;
+import org.apache.usergrid.persistence.collection.serialization.MvccEntitySerializationStrategy;
+import org.apache.usergrid.persistence.collection.serialization.impl.CollectionDataVersions;
+import org.apache.usergrid.persistence.collection.serialization.impl.MvccEntitySerializationStrategyV1Impl;
+import org.apache.usergrid.persistence.collection.serialization.impl.MvccEntitySerializationStrategyV3Impl;
+import org.apache.usergrid.persistence.core.guice.DataMigrationResetRule;
+import org.apache.usergrid.persistence.core.migration.data.DataMigrationManager;
+import org.apache.usergrid.persistence.core.migration.data.TestProgressObserver;
+import org.apache.usergrid.persistence.core.migration.data.newimpls.MigrationDataProvider;
+import org.apache.usergrid.persistence.core.migration.data.newimpls.MigrationRelationship;
+import org.apache.usergrid.persistence.core.migration.data.newimpls.VersionedMigrationSet;
+import org.apache.usergrid.persistence.core.test.ITRunner;
+import org.apache.usergrid.persistence.core.test.UseModules;
+import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.entity.Id;
+import org.apache.usergrid.persistence.model.entity.SimpleId;
+import org.apache.usergrid.persistence.model.util.UUIDGenerator;
+
+import com.google.inject.Inject;
+import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
+
+import net.jcip.annotations.NotThreadSafe;
+
+import rx.Observable;
+
+import static org.apache.usergrid.persistence.core.util.IdGenerator.createId;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+
+
+@NotThreadSafe
+@RunWith( ITRunner.class )
+@UseModules( { TestCollectionModule.class } )
+public class MvccEntityDataMigrationV1ToV3ImplTest implements DataMigrationResetRule.DataMigrationManagerProvider {
+
+ @Inject
+ public DataMigrationManager dataMigrationManager;
+
+ @Inject
+ private MvccEntitySerializationStrategyV1Impl v1Impl;
+
+ @Inject
+ private MvccEntitySerializationStrategyV3Impl v3Impl;
+
+ @Inject
+ public MvccEntityDataMigrationImpl mvccEntityDataMigrationImpl;
+
+ @Inject
+ public VersionedMigrationSet<MvccEntitySerializationStrategy> versions;
+
+ /**
+ * Rule to do the resets we need
+ */
+ @Rule
+ public DataMigrationResetRule migrationTestRule =
+ new DataMigrationResetRule( this, CollectionMigrationPlugin.PLUGIN_NAME,
+ CollectionDataVersions.INITIAL.getVersion() );
+
+
+ @Test
+ public void testMigration() throws ConnectionException {
+
+ final Id applicationId = createId("application");
+ final String collectionName = "things";
+
+ CollectionScope scope = new CollectionScopeImpl(applicationId, applicationId, collectionName );
+
+ final MvccEntity entity1 = getEntity( "thing" );
+ final MvccEntity entity2 = getEntity( "thing" );
+
+ v1Impl.write( scope, entity1 ).execute();
+ v1Impl.write( scope, entity2 ).execute();
+
+
+ MvccEntity returned1 = v1Impl.load( scope, entity1.getId() ).get();
+ MvccEntity returned2 = v1Impl.load( scope, entity2.getId() ).get();
+
+ assertEquals("Same entity", entity1, returned1);
+ assertEquals("Same entity", entity2, returned2);
+
+ final Observable<EntityIdScope> entityIdScope = Observable.from( new EntityIdScope( scope, entity1.getId() ), new EntityIdScope( scope, entity2.getId() ) );
+
+
+ final MigrationDataProvider<EntityIdScope> migrationProvider = new MigrationDataProvider<EntityIdScope>() {
+ @Override
+ public Observable<EntityIdScope> getData() {
+ return entityIdScope;
+ }
+ };
+
+ final TestProgressObserver progressObserver = new TestProgressObserver();
+
+ //now migration
+ final int newVersion = mvccEntityDataMigrationImpl.migrate( CollectionDataVersions.INITIAL.getVersion(), migrationProvider, progressObserver );
+
+
+ assertEquals( "Correct version returned", newVersion, CollectionDataVersions.LOG_REMOVAL.getVersion() );
+ assertFalse( "Progress observer should not have failed", progressObserver.getFailed() );
+ assertTrue( "Progress observer should have update messages", progressObserver.getUpdates().size() > 0 );
+
+
+ //now verify we can read the data correctly in the new version
+ returned1 = v3Impl.load( scope, entity1.getId() ).get();
+ returned2 = v3Impl.load( scope, entity2.getId() ).get();
+
+ assertEquals("Same entity", entity1, returned1);
+ assertEquals("Same entity", entity2, returned2);
+
+ //verify the tuple is correct
+
+ final MigrationRelationship<MvccEntitySerializationStrategy>
+ tuple = versions.getMigrationRelationship( newVersion );
+
+
+ assertSame("Same instance for from", v1Impl, tuple.from);
+ assertSame("Same instance for to", v3Impl, tuple.to);
+
+
+
+ }
+
+
+ private MvccEntity getEntity(final String type){
+
+ final SimpleId entityId = new SimpleId( type );
+ final UUID version = UUIDGenerator.newTimeUUID();
+ final Entity entity = new Entity( entityId );
+
+ MvccEntityImpl logEntry = new MvccEntityImpl( entityId, version, MvccEntity.Status.COMPLETE, entity );
+
+
+ return logEntry;
+
+
+ }
+
+
+
+
+
+ @Override
+ public DataMigrationManager getDataMigrationManager() {
+ return dataMigrationManager;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/802dcdec/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/guice/DataMigrationResetRule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/guice/DataMigrationResetRule.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/guice/DataMigrationResetRule.java
new file mode 100644
index 0000000..0167ff6
--- /dev/null
+++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/guice/DataMigrationResetRule.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence.core.guice;
+
+
+import org.junit.rules.ExternalResource;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.usergrid.persistence.core.migration.data.DataMigrationManager;
+import org.apache.usergrid.persistence.core.migration.schema.MigrationException;
+
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+
+
+/**
+ * A test rule that will set up a specific version for the plugin before test invocation
+ * then set it back afterwards
+ */
+public class DataMigrationResetRule extends ExternalResource {
+ private static final Logger logger = LoggerFactory.getLogger( DataMigrationResetRule.class );
+
+
+ private DataMigrationManagerProvider dataMigrationManagerProvider;
+
+ private final String pluginName;
+
+ private final int versionToSet;
+
+ private int existingVersion = -1;
+
+
+ public DataMigrationResetRule( final DataMigrationManagerProvider dataMigrationManagerProvider, final String pluginName, final int versionToSet ) {
+ this.dataMigrationManagerProvider = dataMigrationManagerProvider;
+ this.pluginName = pluginName;
+ this.versionToSet = versionToSet;
+ }
+
+
+
+ @Override
+ protected void before() throws MigrationException {
+
+ existingVersion = dataMigrationManagerProvider.getDataMigrationManager().getCurrentVersion( pluginName );
+
+ dataMigrationManagerProvider.getDataMigrationManager().resetToVersion( pluginName, versionToSet );
+
+ logger.info( "Migration complete" );
+ }
+
+
+ @Override
+ protected void after() {
+ dataMigrationManagerProvider.getDataMigrationManager().resetToVersion( pluginName, existingVersion );
+ }
+
+
+ /**
+ * Interface for getting a data migration manager during testing. Ugly, but required because we
+ * can't inject into this member
+ */
+ public static interface DataMigrationManagerProvider{
+
+ /**
+ * Get the data migration manager
+ * @return
+ */
+ public DataMigrationManager getDataMigrationManager();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/802dcdec/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/TestProgressObserver.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/TestProgressObserver.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/TestProgressObserver.java
new file mode 100644
index 0000000..63fad42
--- /dev/null
+++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/TestProgressObserver.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence.core.migration.data;
+
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.usergrid.persistence.core.migration.data.newimpls.ProgressObserver;
+
+
+public class TestProgressObserver implements ProgressObserver {
+
+ private boolean failed = false;
+
+
+ private List<String> updates = new ArrayList<>( 100 );
+
+
+ @Override
+ public void failed( final int migrationVersion, final String reason ) {
+ failed = true;
+ }
+
+
+ @Override
+ public void failed( final int migrationVersion, final String reason, final Throwable throwable ) {
+ failed = true;
+ }
+
+
+ @Override
+ public void update( final int migrationVersion, final String message ) {
+ updates.add( message );
+ }
+
+
+ /**
+ * Get if we failed
+ * @return
+ */
+ public boolean getFailed() {
+ return failed;
+ }
+
+
+ /**
+ * Get update messages
+ * @return
+ */
+ public List<String> getUpdates() {
+ return updates;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/802dcdec/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/util/IdGenerator.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/util/IdGenerator.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/util/IdGenerator.java
new file mode 100644
index 0000000..1b117a9
--- /dev/null
+++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/util/IdGenerator.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence.core.util;
+
+
+import java.util.UUID;
+
+import org.apache.usergrid.persistence.model.entity.Id;
+import org.apache.usergrid.persistence.model.entity.SimpleId;
+import org.apache.usergrid.persistence.model.util.UUIDGenerator;
+
+
+/**
+ * Test helper for generating ids
+ */
+public class IdGenerator {
+ /**
+ * Create the id
+ */
+ public static Id createId( String type ) {
+ return createId( UUIDGenerator.newTimeUUID(), type );
+ }
+
+
+ /**
+ * Generate an ID with the type and id
+ *
+ * @param id The uuid in the id
+ * @param type The type of id
+ */
+ public static Id createId( UUID id, String type ) {
+ return new SimpleId( id, type );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/802dcdec/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/guice/GraphModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/guice/GraphModule.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/guice/GraphModule.java
index 67459b2..d504e47 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/guice/GraphModule.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/guice/GraphModule.java
@@ -47,7 +47,7 @@ import org.apache.usergrid.persistence.graph.serialization.EdgeSerialization;
import org.apache.usergrid.persistence.graph.serialization.EdgesObservable;
import org.apache.usergrid.persistence.graph.serialization.NodeSerialization;
import org.apache.usergrid.persistence.graph.serialization.TargetIdObservable;
-import org.apache.usergrid.persistence.graph.serialization.impl.EdgeDataMigrationImpl;
+import org.apache.usergrid.persistence.graph.serialization.impl.migration.EdgeDataMigrationImpl;
import org.apache.usergrid.persistence.graph.serialization.impl.EdgeMetadataSerializationProxyImpl;
import org.apache.usergrid.persistence.graph.serialization.impl.EdgeMetadataSerializationV1Impl;
import org.apache.usergrid.persistence.graph.serialization.impl.EdgeMetadataSerializationV2Impl;
@@ -57,6 +57,7 @@ import org.apache.usergrid.persistence.graph.serialization.impl.GraphManagerFact
import org.apache.usergrid.persistence.graph.serialization.impl.NodeSerializationImpl;
import org.apache.usergrid.persistence.graph.serialization.impl.TargetIdObservableImpl;
import org.apache.usergrid.persistence.graph.serialization.impl.migration.GraphMigrationPlugin;
+import org.apache.usergrid.persistence.graph.serialization.impl.migration.GraphNode;
import org.apache.usergrid.persistence.graph.serialization.impl.shard.EdgeColumnFamilies;
import org.apache.usergrid.persistence.graph.serialization.impl.shard.EdgeShardSerialization;
import org.apache.usergrid.persistence.graph.serialization.impl.shard.EdgeShardStrategy;
@@ -130,8 +131,8 @@ public abstract class GraphModule extends AbstractModule {
//wire up the edg migration
- Multibinder<DataMigration2<ApplicationScope>> dataMigrationMultibinder =
- Multibinder.newSetBinder( binder(), new TypeLiteral<DataMigration2<ApplicationScope>>() {} );
+ Multibinder<DataMigration2<GraphNode>> dataMigrationMultibinder =
+ Multibinder.newSetBinder( binder(), new TypeLiteral<DataMigration2<GraphNode>>() {} );
dataMigrationMultibinder.addBinding().to( EdgeDataMigrationImpl.class );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/802dcdec/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeDataMigrationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeDataMigrationImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeDataMigrationImpl.java
deleted file mode 100644
index 49a954d..0000000
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeDataMigrationImpl.java
+++ /dev/null
@@ -1,152 +0,0 @@
-/*
- *
- * * Licensed to the Apache Software Foundation (ASF) under one or more
- * * contributor license agreements. The ASF licenses this file to You
- * * under the Apache License, Version 2.0 (the "License"); you may not
- * * use this file except in compliance with the License.
- * * You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing, software
- * * distributed under the License is distributed on an "AS IS" BASIS,
- * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * * See the License for the specific language governing permissions and
- * * limitations under the License. For additional information regarding
- * * copyright in this work, please see the NOTICE file in the top level
- * * directory of this distribution.
- *
- */
-package org.apache.usergrid.persistence.graph.serialization.impl;
-
-import com.google.inject.Inject;
-import com.netflix.astyanax.Keyspace;
-import com.netflix.astyanax.MutationBatch;
-import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
-import org.apache.usergrid.persistence.core.migration.data.newimpls.DataMigration2;
-import org.apache.usergrid.persistence.core.migration.data.newimpls.MigrationDataProvider;
-import org.apache.usergrid.persistence.core.migration.data.newimpls.MigrationRelationship;
-import org.apache.usergrid.persistence.core.migration.data.newimpls.ProgressObserver;
-import org.apache.usergrid.persistence.core.migration.data.newimpls.VersionedMigrationSet;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.graph.Edge;
-import org.apache.usergrid.persistence.graph.GraphManager;
-import org.apache.usergrid.persistence.graph.GraphManagerFactory;
-import org.apache.usergrid.persistence.graph.serialization.EdgeMetadataSerialization;
-import org.apache.usergrid.persistence.graph.serialization.EdgesObservable;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import rx.Observable;
-import rx.functions.Action1;
-import rx.functions.Func1;
-import rx.functions.Func2;
-
-import java.util.List;
-import java.util.concurrent.atomic.AtomicLong;
-
-/**
- * Encapsulates the migration of edge meta data
- */
-public class EdgeDataMigrationImpl implements DataMigration2<ApplicationScope> {
-
- private static final Logger logger = LoggerFactory.getLogger(EdgeDataMigrationImpl.class);
-
- private final Keyspace keyspace;
- private final GraphManagerFactory graphManagerFactory;
- private final EdgesObservable edgesFromSourceObservable;
- private final VersionedMigrationSet<EdgeMetadataSerialization> allVersions;
- private final EdgeMetadataSerializationV2Impl edgeMetadataSerializationV2;
-
- @Inject
- public EdgeDataMigrationImpl( final Keyspace keyspace, final GraphManagerFactory graphManagerFactory,
- final EdgesObservable edgesFromSourceObservable,
-
- final VersionedMigrationSet<EdgeMetadataSerialization> allVersions,
- final EdgeMetadataSerializationV2Impl edgeMetadataSerializationV2 ) {
-
- this.keyspace = keyspace;
- this.graphManagerFactory = graphManagerFactory;
- this.edgesFromSourceObservable = edgesFromSourceObservable;
- this.allVersions = allVersions;
- this.edgeMetadataSerializationV2 = edgeMetadataSerializationV2;
- }
-
-
-
-
- @Override
- public int migrate( final int currentVersion, final MigrationDataProvider<ApplicationScope> migrationDataProvider,
- final ProgressObserver observer ) {
-
- final AtomicLong counter = new AtomicLong();
-
- final MigrationRelationship<EdgeMetadataSerialization>
- migration = allVersions.getMigrationRelationship( currentVersion );
-
- final Observable<List<Edge>> observable = migrationDataProvider.getData().flatMap(new Func1<ApplicationScope, Observable<List<Edge>>>() {
- @Override
- public Observable<List<Edge>> call(final ApplicationScope applicationScope) {
- final GraphManager gm = graphManagerFactory.createEdgeManager( applicationScope );
- final Observable<Edge> edgesFromSource =
- edgesFromSourceObservable.edgesFromSource( gm, applicationScope.getApplication() );
- logger.info( "Migrating edges scope {}", applicationScope );
-
- //get each edge from this node as a source
- return edgesFromSource
-
- //for each edge, re-index it in v2 every 1000 edges or less
- .buffer( 1000 ).parallel( new Func1<Observable<List<Edge>>, Observable<List<Edge>>>() {
- @Override
- public Observable<List<Edge>> call( final Observable<List<Edge>> listObservable ) {
- return listObservable.doOnNext( new Action1<List<Edge>>() {
- @Override
- public void call( List<Edge> edges ) {
- final MutationBatch batch = keyspace.prepareMutationBatch();
-
- for ( Edge edge : edges ) {
- logger.info( "Migrating meta for edge {}", edge );
- final MutationBatch edgeBatch =
- migration.to.writeEdge( applicationScope, edge );
- batch.mergeShallow( edgeBatch );
- }
-
- try {
- batch.execute();
- }
- catch ( ConnectionException e ) {
- throw new RuntimeException( "Unable to perform migration", e );
- }
-
- //update the observer so the admin can see it
- final long newCount = counter.addAndGet( edges.size() );
-
- observer.update( migration.to.getImplementationVersion(),
- String.format( "Currently running. Rewritten %d edge types",
- newCount ) );
- }
- } );
- }
- } );
- }});
-
- observable.longCount().toBlocking().last();
-
- return migration.to.getImplementationVersion();
-
- }
-
-
-
-
- @Override
- public boolean supports( final int currentVersion ) {
- return currentVersion < edgeMetadataSerializationV2.getImplementationVersion();
- }
-
-
- @Override
- public int getMaxVersion() {
- //we only support up to v2 ATM
- return edgeMetadataSerializationV2.getImplementationVersion();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/802dcdec/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/EdgeDataMigrationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/EdgeDataMigrationImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/EdgeDataMigrationImpl.java
new file mode 100644
index 0000000..af157a6
--- /dev/null
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/EdgeDataMigrationImpl.java
@@ -0,0 +1,149 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. The ASF licenses this file to You
+ * * under the Apache License, Version 2.0 (the "License"); you may not
+ * * use this file except in compliance with the License.
+ * * You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License. For additional information regarding
+ * * copyright in this work, please see the NOTICE file in the top level
+ * * directory of this distribution.
+ *
+ */
+package org.apache.usergrid.persistence.graph.serialization.impl.migration;
+
+import com.google.inject.Inject;
+import com.netflix.astyanax.Keyspace;
+import com.netflix.astyanax.MutationBatch;
+import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
+import org.apache.usergrid.persistence.core.migration.data.newimpls.DataMigration2;
+import org.apache.usergrid.persistence.core.migration.data.newimpls.MigrationDataProvider;
+import org.apache.usergrid.persistence.core.migration.data.newimpls.MigrationRelationship;
+import org.apache.usergrid.persistence.core.migration.data.newimpls.ProgressObserver;
+import org.apache.usergrid.persistence.core.migration.data.newimpls.VersionedMigrationSet;
+import org.apache.usergrid.persistence.graph.Edge;
+import org.apache.usergrid.persistence.graph.GraphManager;
+import org.apache.usergrid.persistence.graph.GraphManagerFactory;
+import org.apache.usergrid.persistence.graph.serialization.EdgeMetadataSerialization;
+import org.apache.usergrid.persistence.graph.serialization.EdgesObservable;
+import org.apache.usergrid.persistence.graph.serialization.impl.EdgeMetadataSerializationV2Impl;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import rx.Observable;
+import rx.functions.Action1;
+import rx.functions.Func1;
+
+import java.util.List;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * Encapsulates the migration of edge meta data.
+ *
+ * The migration data provider assumes that is will visit every node in the graph
+ * all edges from these source node will then be re-indexed.
+ */
+public class EdgeDataMigrationImpl implements DataMigration2<GraphNode> {
+
+ private static final Logger logger = LoggerFactory.getLogger(EdgeDataMigrationImpl.class);
+
+ private final Keyspace keyspace;
+ private final GraphManagerFactory graphManagerFactory;
+ private final EdgesObservable edgesFromSourceObservable;
+ private final VersionedMigrationSet<EdgeMetadataSerialization> allVersions;
+ private final EdgeMetadataSerializationV2Impl edgeMetadataSerializationV2;
+
+ @Inject
+ public EdgeDataMigrationImpl( final Keyspace keyspace,
+ final GraphManagerFactory graphManagerFactory,
+ final EdgesObservable edgesFromSourceObservable,
+ final VersionedMigrationSet<EdgeMetadataSerialization> allVersions,
+ final EdgeMetadataSerializationV2Impl edgeMetadataSerializationV2 ) {
+
+ this.keyspace = keyspace;
+ this.graphManagerFactory = graphManagerFactory;
+ this.edgesFromSourceObservable = edgesFromSourceObservable;
+ this.allVersions = allVersions;
+ this.edgeMetadataSerializationV2 = edgeMetadataSerializationV2;
+ }
+
+
+
+
+ @Override
+ public int migrate( final int currentVersion, final MigrationDataProvider<GraphNode> migrationDataProvider,
+ final ProgressObserver observer ) {
+
+ final AtomicLong counter = new AtomicLong();
+
+ final MigrationRelationship<EdgeMetadataSerialization>
+ migration = allVersions.getMigrationRelationship( currentVersion );
+
+ final Observable<List<Edge>> observable = migrationDataProvider.getData().flatMap( new Func1<GraphNode,
+ Observable<List<Edge>>>() {
+ @Override
+ public Observable<List<Edge>> call( final GraphNode graphNode ) {
+ final GraphManager gm = graphManagerFactory.createEdgeManager( graphNode.applicationScope );
+
+ //get edges from the source
+ return edgesFromSourceObservable.edgesFromSource( gm, graphNode.entryNode ).buffer( 1000 ).parallel( new Func1<Observable<List<Edge>>, Observable<List<Edge>>>() {
+ @Override
+ public Observable<List<Edge>> call( final Observable<List<Edge>> listObservable ) {
+ return listObservable.doOnNext( new Action1<List<Edge>>() {
+ @Override
+ public void call( List<Edge> edges ) {
+ final MutationBatch batch = keyspace.prepareMutationBatch();
+
+ for ( Edge edge : edges ) {
+ logger.info( "Migrating meta for edge {}", edge );
+ final MutationBatch edgeBatch =
+ migration.to.writeEdge( graphNode.applicationScope, edge );
+ batch.mergeShallow( edgeBatch );
+ }
+
+ try {
+ batch.execute();
+ }
+ catch ( ConnectionException e ) {
+ throw new RuntimeException( "Unable to perform migration", e );
+ }
+
+ //update the observer so the admin can see it
+ final long newCount = counter.addAndGet( edges.size() );
+
+ observer.update( migration.to.getImplementationVersion(),
+ String.format( "Currently running. Rewritten %d edge types",
+ newCount ) );
+ }
+ } );
+ } } );
+ }} );
+
+ observable.longCount().toBlocking().last();
+
+ return migration.to.getImplementationVersion();
+
+ }
+
+
+
+
+ @Override
+ public boolean supports( final int currentVersion ) {
+ return currentVersion < edgeMetadataSerializationV2.getImplementationVersion();
+ }
+
+
+ @Override
+ public int getMaxVersion() {
+ //we only support up to v2 ATM
+ return edgeMetadataSerializationV2.getImplementationVersion();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/802dcdec/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/GraphMigrationPlugin.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/GraphMigrationPlugin.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/GraphMigrationPlugin.java
index 0d3405b..c989822 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/GraphMigrationPlugin.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/GraphMigrationPlugin.java
@@ -30,7 +30,6 @@ import org.apache.usergrid.persistence.core.migration.data.MigrationInfoSerializ
import org.apache.usergrid.persistence.core.migration.data.newimpls.AbstractMigrationPlugin;
import org.apache.usergrid.persistence.core.migration.data.newimpls.DataMigration2;
import org.apache.usergrid.persistence.core.migration.data.newimpls.MigrationDataProvider;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import com.google.inject.Inject;
import com.google.inject.Singleton;
@@ -40,15 +39,15 @@ import com.google.inject.Singleton;
* Migration plugin for the collection module
*/
@Singleton
-public class GraphMigrationPlugin extends AbstractMigrationPlugin<ApplicationScope> {
+public class GraphMigrationPlugin extends AbstractMigrationPlugin<GraphNode> {
public static final String PLUGIN_NAME = "graph-data";
@Inject
- public GraphMigrationPlugin( final Set<DataMigration2<ApplicationScope>> entityDataMigrations,
- final MigrationDataProvider<ApplicationScope> entityIdScopeDataMigrationProvider,
+ public GraphMigrationPlugin( final Set<DataMigration2<GraphNode>> entityDataMigrations,
+ final MigrationDataProvider<GraphNode> entityIdScopeDataMigrationProvider,
final MigrationInfoSerialization migrationInfoSerialization ) {
super( entityDataMigrations, entityIdScopeDataMigrationProvider, migrationInfoSerialization );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/802dcdec/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/GraphNode.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/GraphNode.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/GraphNode.java
new file mode 100644
index 0000000..00aa617
--- /dev/null
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/GraphNode.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence.graph.serialization.impl.migration;
+
+
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+
+/**
+ * Traverses a graph give the scope and then will traverse from the root node to the last scope.
+ */
+public class GraphNode {
+ public final ApplicationScope applicationScope;
+ public final Id entryNode;
+
+
+ public GraphNode( final ApplicationScope applicationScope, final Id entryNode ) {
+ this.applicationScope = applicationScope;
+ this.entryNode = entryNode;
+ }
+}
[22/50] [abbrv] incubator-usergrid git commit: Fixes incorrect system
initialization order
Posted by sf...@apache.org.
Fixes incorrect system initialization order
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/494f8aed
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/494f8aed
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/494f8aed
Branch: refs/heads/USERGRID-480
Commit: 494f8aed2ad970cae9ff64433e5229b0b67d965a
Parents: 2e77c63
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Mar 4 22:32:07 2015 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Mar 4 22:32:07 2015 -0700
----------------------------------------------------------------------
.../main/dist/init_instance/init_rest_server.sh | 24 +++++++++++---------
1 file changed, 13 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/494f8aed/stack/awscluster/src/main/dist/init_instance/init_rest_server.sh
----------------------------------------------------------------------
diff --git a/stack/awscluster/src/main/dist/init_instance/init_rest_server.sh b/stack/awscluster/src/main/dist/init_instance/init_rest_server.sh
index 3860cd4..0b920fd 100644
--- a/stack/awscluster/src/main/dist/init_instance/init_rest_server.sh
+++ b/stack/awscluster/src/main/dist/init_instance/init_rest_server.sh
@@ -1,14 +1,14 @@
#!/bin/bash
-#
+#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. The ASF licenses this file to You
# under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License.
# You may obtain a copy of the License at
-#
+#
# http://www.apache.org/licenses/LICENSE-2.0
-#
+#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -35,10 +35,10 @@ apt-get -y --force-yes install ${PKGS}
# Install AWS Java SDK and get it into the Groovy classpath
curl http://sdk-for-java.amazonwebservices.com/latest/aws-java-sdk.zip > /tmp/aws-sdk-java.zip
cd /usr/share/
-unzip /tmp/aws-sdk-java.zip
+unzip /tmp/aws-sdk-java.zip
mkdir -p /home/ubuntu/.groovy/lib
cp /usr/share/aws-java-sdk-*/third-party/*/*.jar /home/ubuntu/.groovy/lib
-cp /usr/share/aws-java-sdk-*/lib/* /home/ubuntu/.groovy/lib
+cp /usr/share/aws-java-sdk-*/lib/* /home/ubuntu/.groovy/lib
ln -s /home/ubuntu/.groovy /root/.groovy
# Build environment for Groovy scripts
@@ -55,7 +55,7 @@ groovy tag_instance.groovy -BUILD-IN-PROGRESS
chmod +x /usr/share/usergrid/update.sh
cd /usr/share/usergrid/init_instance
-./install_oraclejdk.sh
+./install_oraclejdk.sh
cd /usr/share/usergrid/init_instance
./install_yourkit.sh
@@ -171,7 +171,7 @@ kernel.shmall = 4294967296
######
EOF
-# wait for enough Cassandra nodes then delpoy and configure Usergrid
+# wait for enough Cassandra nodes then delpoy and configure Usergrid
cd /usr/share/usergrid/scripts
groovy wait_for_instances.groovy cassandra ${CASSANDRA_NUM_SERVERS}
groovy wait_for_instances.groovy elasticsearch ${ES_NUM_SERVERS}
@@ -185,8 +185,8 @@ chown -R tomcat7 /usr/share/usergrid/webapps
chown -R tomcat7 /var/lib/tomcat7/webapps
# configure usergrid
-mkdir -p /usr/share/tomcat7/lib
-groovy configure_usergrid.groovy > /usr/share/tomcat7/lib/usergrid-deployment.properties
+mkdir -p /usr/share/tomcat7/lib
+groovy configure_usergrid.groovy > /usr/share/tomcat7/lib/usergrid-deployment.properties
groovy configure_portal_new.groovy >> /var/lib/tomcat7/webapps/portal/config.js
@@ -222,6 +222,10 @@ GRAPHITE_SERVER="$(groovy get_first_instance.groovy graphite )"
#First host run the migration and setup
if [ "$FIRSTHOST"=="$PUBLIC_HOSTNAME" ]; then
+#Run the system database setup since migration is a no-op
+curl -X GET http://localhost:8080/system/database/setup -u superuser:test
+
+
#Run the migration
curl -X PUT http://localhost:8080/system/migrate/run -u superuser:test
@@ -229,8 +233,6 @@ curl -X PUT http://localhost:8080/system/migrate/run -u superuser:test
#Running setup
sleep 10
-#Run the system database setup since migration is a no-op
-curl -X GET http://localhost:8080/system/database/setup -u superuser:test
cd /usr/share/usergrid/init_instance
./update_keyspaces.sh
[06/50] [abbrv] incubator-usergrid git commit: Temporary commit,
still refactoring modules and Guice configuration.
Posted by sf...@apache.org.
Temporary commit, still refactoring modules and Guice configuration.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/e96af464
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/e96af464
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/e96af464
Branch: refs/heads/USERGRID-480
Commit: e96af464bc345d0b718fb34e06774c053af9cb98
Parents: d3f8ee6
Author: Todd Nine <tn...@apigee.com>
Authored: Fri Feb 27 14:50:10 2015 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Fri Feb 27 14:50:10 2015 -0700
----------------------------------------------------------------------
.../collection/guice/CollectionModule.java | 4 +-
.../mvcc/stage/write/WriteCommit.java | 5 +-
.../impl/CollectionDataVersions.java | 44 ++
...vccEntitySerializationStrategyProxyImpl.java | 112 +++--
.../MvccEntitySerializationStrategyV1Impl.java | 6 +
.../MvccEntitySerializationStrategyV2Impl.java | 6 +
.../MvccEntitySerializationStrategyV3Impl.java | 6 +
.../serialization/impl/SerializationModule.java | 75 +++-
.../migration/CollectionMigrationPlugin.java | 9 +-
.../migration/MvccEntityDataMigrationImpl.java | 181 +++++---
.../collection/util/EntityUtils.java | 29 +-
.../collection/guice/TestCollectionModule.java | 12 +-
...ntitySerializationStrategyProxyV1_2Test.java | 82 ----
...ntitySerializationStrategyProxyV1_3Test.java | 28 +-
...cEntitySerializationStrategyProxyV2Test.java | 76 ----
...ntitySerializationStrategyProxyV2_3Test.java | 81 ++++
...ccEntitySerializationStrategyV1ImplTest.java | 10 +-
...ccEntitySerializationStrategyV2ImplTest.java | 8 +-
...ccEntitySerializationStrategyV3ImplTest.java | 4 +-
.../usergrid/persistence/core/guice/V1Impl.java | 37 --
.../persistence/core/guice/V1ProxyImpl.java | 35 --
.../usergrid/persistence/core/guice/V2Impl.java | 39 --
.../usergrid/persistence/core/guice/V3Impl.java | 39 --
.../data/ApplicationDataMigration.java | 32 --
.../migration/data/CollectionDataMigration.java | 30 --
.../migration/data/DataMigrationManager.java | 5 -
.../data/DataMigrationManagerImpl.java | 17 -
.../core/migration/data/MigrationInfoCache.java | 52 +++
.../migration/data/MigrationInfoCacheImpl.java | 84 ++++
.../data/newimpls/MigrationRelationship.java | 61 ++-
.../data/newimpls/VersionedMigrationSet.java | 153 +++++++
.../migration/data/newimpls/VersionedSet.java | 113 -----
.../core/rx/AllEntitiesInSystemObservable.java | 53 ---
.../core/rx/ApplicationObservable.java | 39 --
.../core/guice/MaxMigrationModule.java | 40 --
.../core/guice/MaxMigrationVersion.java | 49 --
.../core/guice/TestCommonModule.java | 8 -
.../persistence/core/guice/TestModule.java | 2 -
.../data/DataMigrationManagerImplTest.java | 443 +++++++++----------
.../data/MigrationInfoSerializationTest.java | 104 ++---
.../newimpls/VersionedMigrationSetTest.java | 194 ++++++++
.../rx/AllEntitiesInSystemTestObservable.java | 39 --
.../core/rx/ApplicationsTestObservable.java | 39 --
43 files changed, 1261 insertions(+), 1224 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e96af464/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/guice/CollectionModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/guice/CollectionModule.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/guice/CollectionModule.java
index 8fda6ca..af5c4cf 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/guice/CollectionModule.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/guice/CollectionModule.java
@@ -78,7 +78,7 @@ public class CollectionModule extends AbstractModule {
Multibinder.newSetBinder( binder(), EntityDeleted.class );
// create a guice factor for getting our collection manager
- bind(EntityCollectionManagerFactory.class).to(EntityCollectionManagerFactoryImpl.class);
+ bind(EntityCollectionManagerFactory.class).to(EntityCollectionManagerFactoryImpl.class);
//bind this to our factory
install( new GuicyFigModule( EntityCacheFig.class ) );
@@ -98,6 +98,8 @@ public class CollectionModule extends AbstractModule {
return writeStart;
}
+
+ //TODO USERGRID-405, remove this, it's no longer supported
@Provides
@Singleton
@Inject
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e96af464/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteCommit.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteCommit.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteCommit.java
index 1a90ed8..8889b2f 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteCommit.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteCommit.java
@@ -111,9 +111,7 @@ public class WriteCommit implements Func1<CollectionIoEvent<MvccEntity>, Entity>
logMutation.mergeShallow( entityMutation );
// re-write the unique values but this time with no TTL
- for ( Field field : mvccEntity.getEntity().get().getFields() ) {
-
- if ( field.isUnique() ) {
+ for ( Field field : EntityUtils.getUniqueFields(mvccEntity.getEntity().get()) ) {
UniqueValue written = new UniqueValueImpl( field,
entityId,version);
@@ -124,7 +122,6 @@ public class WriteCommit implements Func1<CollectionIoEvent<MvccEntity>, Entity>
// merge into our existing mutation batch
logMutation.mergeShallow( mb );
- }
}
try {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e96af464/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/CollectionDataVersions.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/CollectionDataVersions.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/CollectionDataVersions.java
new file mode 100644
index 0000000..ee84b1e
--- /dev/null
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/CollectionDataVersions.java
@@ -0,0 +1,44 @@
+/*
+ *
+ * *
+ * * Licensed to the Apache Software Foundation (ASF) under one
+ * * or more contributor license agreements. See the NOTICE file
+ * * distributed with this work for additional information
+ * * regarding copyright ownership. The ASF licenses this file
+ * * to you under the Apache License, Version 2.0 (the
+ * * "License"); you may not use this file except in compliance
+ * * with the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing,
+ * * software distributed under the License is distributed on an
+ * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * * KIND, either express or implied. See the License for the
+ * * specific language governing permissions and limitations
+ * * under the License.
+ * *
+ *
+ */
+
+package org.apache.usergrid.persistence.collection.serialization.impl;
+
+
+/**
+ * Versions of data as they exist across our system
+ */
+public enum CollectionDataVersions{
+ ONE(1),
+ TWO(2),
+ THREE(3);
+
+ private final int version;
+
+
+ private CollectionDataVersions( final int version ) {this.version = version;}
+
+
+ public int getVersion() {
+ return version;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e96af464/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyImpl.java
index 03271a2..02dfd9d 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyImpl.java
@@ -16,21 +16,27 @@
*/
package org.apache.usergrid.persistence.collection.serialization.impl;
-import com.google.common.base.Optional;
-import com.google.inject.Inject;
-import com.netflix.astyanax.Keyspace;
-import com.netflix.astyanax.MutationBatch;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.UUID;
+
import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.collection.EntitySet;
import org.apache.usergrid.persistence.collection.MvccEntity;
-import org.apache.usergrid.persistence.collection.mvcc.MvccEntityMigrationStrategy;
import org.apache.usergrid.persistence.collection.serialization.MvccEntitySerializationStrategy;
+import org.apache.usergrid.persistence.collection.serialization.impl.migration.CollectionMigrationPlugin;
import org.apache.usergrid.persistence.core.astyanax.MultiTennantColumnFamilyDefinition;
-import org.apache.usergrid.persistence.core.migration.data.DataMigrationManager;
-import org.apache.usergrid.persistence.core.migration.data.MigrationInfoSerialization;
+import org.apache.usergrid.persistence.core.migration.data.MigrationInfoCache;
+import org.apache.usergrid.persistence.core.migration.data.newimpls.MigrationRelationship;
+import org.apache.usergrid.persistence.core.migration.data.newimpls.VersionedMigrationSet;
import org.apache.usergrid.persistence.model.entity.Id;
-import java.util.*;
+import com.google.common.base.Optional;
+import com.google.inject.Inject;
+import com.netflix.astyanax.Keyspace;
+import com.netflix.astyanax.MutationBatch;
/**
@@ -42,43 +48,49 @@ public class MvccEntitySerializationStrategyProxyImpl implements MvccEntitySeria
protected final Keyspace keyspace;
- private final MigrationInfoSerialization migrationInfoSerialization;
- private final
+ private final VersionedMigrationSet<MvccEntitySerializationStrategy> versions;
+ private final MigrationInfoCache migrationInfoCache;
@Inject
- public MvccEntitySerializationStrategyProxyImpl(final Keyspace keyspace,
- final Set<MvccEntitySerializationStrategy> allVersions,
- final MigrationInfoSerialization migrationInfoSerialization) {
+ public MvccEntitySerializationStrategyProxyImpl( final Keyspace keyspace,
+ final VersionedMigrationSet<MvccEntitySerializationStrategy> allVersions,
+ final MigrationInfoCache migrationInfoCache ) {
this.keyspace = keyspace;
-
- this.migrationInfoSerialization = migrationInfoSerialization;
+ this.migrationInfoCache = migrationInfoCache;
+ this.versions = allVersions;
}
@Override
public MutationBatch write( final CollectionScope context, final MvccEntity entity ) {
- if ( isOldVersion() ) {
+
+ final MigrationRelationship<MvccEntitySerializationStrategy> migration = getMigrationRelationShip();
+
+ if ( migration.needsMigration() ) {
final MutationBatch aggregateBatch = keyspace.prepareMutationBatch();
- aggregateBatch.mergeShallow( previous.write( context, entity ) );
- aggregateBatch.mergeShallow( current.write( context, entity ) );
+ aggregateBatch.mergeShallow( migration.from.write( context, entity ) );
+ aggregateBatch.mergeShallow( migration.to.write( context, entity ) );
return aggregateBatch;
}
- return current.write( context, entity );
+ return migration.to.write( context, entity );
}
@Override
public EntitySet load( final CollectionScope scope, final Collection<Id> entityIds, final UUID maxVersion ) {
- if ( isOldVersion() ) {
- return previous.load( scope, entityIds, maxVersion );
- }
- return current.load( scope, entityIds, maxVersion );
+ final MigrationRelationship<MvccEntitySerializationStrategy> migration = getMigrationRelationShip();
+
+ if ( migration.needsMigration() ) {
+ return migration.from.load( scope, entityIds, maxVersion );
+ }
+
+ return migration.to.load( scope, entityIds, maxVersion );
}
@@ -86,68 +98,84 @@ public class MvccEntitySerializationStrategyProxyImpl implements MvccEntitySeria
@Override
public Iterator<MvccEntity> loadDescendingHistory( final CollectionScope context, final Id entityId,
final UUID version, final int fetchSize ) {
- if ( isOldVersion() ) {
- return previous.loadDescendingHistory( context, entityId, version, fetchSize );
+
+ final MigrationRelationship<MvccEntitySerializationStrategy> migration = getMigrationRelationShip();
+
+ if ( migration.needsMigration()) {
+ return migration.from.loadDescendingHistory( context, entityId, version, fetchSize );
}
- return current.loadDescendingHistory( context, entityId, version, fetchSize );
+ return migration.to.loadDescendingHistory( context, entityId, version, fetchSize );
}
@Override
public Iterator<MvccEntity> loadAscendingHistory( final CollectionScope context, final Id entityId,
final UUID version, final int fetchSize ) {
- if ( isOldVersion() ) {
- return previous.loadAscendingHistory( context, entityId, version, fetchSize );
+
+ final MigrationRelationship<MvccEntitySerializationStrategy> migration = getMigrationRelationShip();
+
+ if ( migration.needsMigration() ) {
+ return migration.from.loadAscendingHistory( context, entityId, version, fetchSize );
}
- return current.loadAscendingHistory( context, entityId, version, fetchSize );
+ return migration.to.loadAscendingHistory( context, entityId, version, fetchSize );
}
@Override
public Optional<MvccEntity> load( final CollectionScope scope, final Id entityId ) {
- if ( isOldVersion() ) {
- return previous.load( scope, entityId );
+
+ final MigrationRelationship<MvccEntitySerializationStrategy> migration = getMigrationRelationShip();
+
+ if ( migration.needsMigration() ) {
+ return migration.from.load( scope, entityId );
}
- return current.load( scope, entityId );
+ return migration.to.load( scope, entityId );
}
@Override
public MutationBatch mark( final CollectionScope context, final Id entityId, final UUID version ) {
- if ( isOldVersion() ) {
+
+ final MigrationRelationship<MvccEntitySerializationStrategy> migration = getMigrationRelationShip();
+
+ if ( migration.needsMigration() ) {
final MutationBatch aggregateBatch = keyspace.prepareMutationBatch();
- aggregateBatch.mergeShallow( previous.mark( context, entityId, version ) );
- aggregateBatch.mergeShallow( current.mark( context, entityId, version ) );
+ aggregateBatch.mergeShallow( migration.from.mark( context, entityId, version ) );
+ aggregateBatch.mergeShallow( migration.to.mark( context, entityId, version ) );
return aggregateBatch;
}
- return current.mark( context, entityId, version );
+ return migration.to.mark( context, entityId, version );
}
@Override
public MutationBatch delete( final CollectionScope context, final Id entityId, final UUID version ) {
- if ( isOldVersion() ) {
+
+ final MigrationRelationship<MvccEntitySerializationStrategy> migration = getMigrationRelationShip();
+
+ if ( migration.needsMigration() ) {
final MutationBatch aggregateBatch = keyspace.prepareMutationBatch();
- aggregateBatch.mergeShallow( previous.delete( context, entityId, version ) );
- aggregateBatch.mergeShallow( current.delete( context, entityId, version ) );
+ aggregateBatch.mergeShallow( migration.from.delete( context, entityId, version ) );
+ aggregateBatch.mergeShallow( migration.to.delete( context, entityId, version ) );
return aggregateBatch;
}
- return current.delete( context, entityId, version );
+ return migration.to.delete( context, entityId, version );
}
/**
* Return true if we're on an old version
*/
- private boolean isOldVersion() {
- return migrationInfoSerialization.getCurrentVersion() < getVersion();
+ private MigrationRelationship<MvccEntitySerializationStrategy> getMigrationRelationShip() {
+ return this.versions.getMigrationRelationship(
+ migrationInfoCache.getVersion( CollectionMigrationPlugin.PLUGIN_NAME ) );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e96af464/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV1Impl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV1Impl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV1Impl.java
index 119fb6d..a959508 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV1Impl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV1Impl.java
@@ -88,6 +88,12 @@ public class MvccEntitySerializationStrategyV1Impl extends MvccEntitySerializati
}
+ @Override
+ public int getImplementationVersion() {
+ return CollectionDataVersions.ONE.getVersion();
+ }
+
+
public static class EntitySerializer extends AbstractSerializer<EntityWrapper> {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e96af464/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2Impl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2Impl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2Impl.java
index cd46c1e..04a1fea 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2Impl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2Impl.java
@@ -91,6 +91,12 @@ public class MvccEntitySerializationStrategyV2Impl extends MvccEntitySerializati
}
+ @Override
+ public int getImplementationVersion() {
+ return CollectionDataVersions.TWO.getVersion();
+ }
+
+
/**
* We should only ever create this once, since this impl is a singleton
*/
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e96af464/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3Impl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3Impl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3Impl.java
index 6b60c89..4e73119 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3Impl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3Impl.java
@@ -381,6 +381,12 @@ public class MvccEntitySerializationStrategyV3Impl implements MvccEntitySerializ
}
+ @Override
+ public int getImplementationVersion() {
+ return CollectionDataVersions.THREE.getVersion();
+ }
+
+
/**
* Simple callback to perform puts and deletes with a common row setup code
*/
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e96af464/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/SerializationModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/SerializationModule.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/SerializationModule.java
index a9c493f..fe804f3 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/SerializationModule.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/SerializationModule.java
@@ -18,23 +18,25 @@
package org.apache.usergrid.persistence.collection.serialization.impl;
-import org.apache.usergrid.persistence.collection.mvcc.MvccEntityMigrationStrategy;
import org.apache.usergrid.persistence.collection.mvcc.MvccLogEntrySerializationStrategy;
import org.apache.usergrid.persistence.collection.serialization.MvccEntitySerializationStrategy;
import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
import org.apache.usergrid.persistence.collection.serialization.impl.migration.CollectionMigrationPlugin;
+import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
import org.apache.usergrid.persistence.collection.serialization.impl.migration.MvccEntityDataMigrationImpl;
import org.apache.usergrid.persistence.core.guice.ProxyImpl;
-import org.apache.usergrid.persistence.core.guice.V1Impl;
-import org.apache.usergrid.persistence.core.guice.V1ProxyImpl;
-import org.apache.usergrid.persistence.core.guice.V2Impl;
-import org.apache.usergrid.persistence.core.guice.V3Impl;
import org.apache.usergrid.persistence.core.migration.data.newimpls.DataMigration2;
import org.apache.usergrid.persistence.core.migration.data.newimpls.MigrationPlugin;
+import org.apache.usergrid.persistence.core.migration.data.newimpls.MigrationRelationship;
+import org.apache.usergrid.persistence.core.migration.data.newimpls.VersionedMigrationSet;
import org.apache.usergrid.persistence.core.migration.schema.Migration;
import com.google.inject.AbstractModule;
+import com.google.inject.Inject;
import com.google.inject.Key;
+import com.google.inject.Provides;
+import com.google.inject.Singleton;
+import com.google.inject.TypeLiteral;
import com.google.inject.multibindings.Multibinder;
@@ -50,25 +52,27 @@ public class SerializationModule extends AbstractModule {
// bind the serialization strategies
//We've migrated this one, so we need to set up the previous, current, and proxy
- bind( MvccEntitySerializationStrategy.class ).annotatedWith( V1Impl.class )
- .to( MvccEntitySerializationStrategyV1Impl.class );
- bind( MvccEntitySerializationStrategy.class ).annotatedWith( V2Impl.class )
- .to(MvccEntitySerializationStrategyV2Impl.class);
- bind( MvccEntitySerializationStrategy.class ).annotatedWith( V3Impl.class )
- .to(MvccEntitySerializationStrategyV3Impl.class);
- bind(MvccEntitySerializationStrategy.class).annotatedWith(V1ProxyImpl.class)
- .to(MvccEntitySerializationStrategyProxyV1Impl.class);
- bind(MvccEntitySerializationStrategy.class ).annotatedWith(ProxyImpl.class)
- .to(MvccEntitySerializationStrategyProxyV2Impl.class);
+ bind(MvccEntitySerializationStrategy.class).annotatedWith( ProxyImpl.class ).to( MvccEntitySerializationStrategyProxyImpl.class );
- Multibinder<DataMigration2> dataMigrationMultibinder = Multibinder.newSetBinder( binder(), DataMigration2.class );
+
+ //bind all 3 implementations
+ bind(MvccEntitySerializationStrategyV1Impl.class);
+ bind(MvccEntitySerializationStrategyV2Impl.class);
+ bind(MvccEntitySerializationStrategyV3Impl.class);
+
+
+
+ //migrations
+ //we want to make sure our generics are retained, so we use a typeliteral
+ Multibinder<DataMigration2<EntityIdScope>> dataMigrationMultibinder = Multibinder.newSetBinder( binder(), new TypeLiteral<DataMigration2<EntityIdScope>>(){});
dataMigrationMultibinder.addBinding().to( MvccEntityDataMigrationImpl.class );
+ //wire up the collection migration plugin
Multibinder.newSetBinder( binder(), MigrationPlugin.class).addBinding().to( CollectionMigrationPlugin.class );
@@ -78,9 +82,9 @@ public class SerializationModule extends AbstractModule {
//do multibindings for migrations
Multibinder<Migration> uriBinder = Multibinder.newSetBinder( binder(), Migration.class );
- uriBinder.addBinding().to( Key.get( MvccEntitySerializationStrategy.class, V1Impl.class ) );
- uriBinder.addBinding().to( Key.get( MvccEntitySerializationStrategy.class, V2Impl.class ) );
- uriBinder.addBinding().to( Key.get( MvccEntitySerializationStrategy.class, V3Impl.class ) );
+ uriBinder.addBinding().to( Key.get( MvccEntitySerializationStrategyV1Impl.class ) );
+ uriBinder.addBinding().to( Key.get( MvccEntitySerializationStrategyV2Impl.class ) );
+ uriBinder.addBinding().to( Key.get( MvccEntitySerializationStrategyV3Impl.class ) );
uriBinder.addBinding().to( Key.get( MvccLogEntrySerializationStrategy.class ) );
uriBinder.addBinding().to( Key.get( UniqueValueSerializationStrategy.class ) );
@@ -88,4 +92,37 @@ public class SerializationModule extends AbstractModule {
//bind our settings as an eager singleton so it's checked on startup
bind(SettingsValidation.class).asEagerSingleton();
}
+
+
+ /**
+ * Configure via explicit declaration the migration path we can follow
+ * @param v1
+ * @param v2
+ * @param v3
+ * @return
+ */
+ @Singleton
+ @Inject
+ @Provides
+ public VersionedMigrationSet<MvccEntitySerializationStrategy> getVersions(final MvccEntitySerializationStrategyV1Impl v1, final MvccEntitySerializationStrategyV2Impl v2, final MvccEntitySerializationStrategyV3Impl v3){
+
+
+ //we must perform a migration from v1 to v3 in order to maintain consistency
+ MigrationRelationship<MvccEntitySerializationStrategy> v1Tov3 = new MigrationRelationship<>( v1, v3 );
+
+ //we must migrate from 2 to 3, this is a bridge that must happen to maintain data consistency
+
+ MigrationRelationship<MvccEntitySerializationStrategy> v2Tov3 = new MigrationRelationship<>( v2, v3 );
+
+
+ //note that we MUST migrate to v3 before our next migration, if v4 and v5 is implemented we will need a v3->v5 and a v4->v5 set
+ MigrationRelationship<MvccEntitySerializationStrategy> current = new MigrationRelationship<MvccEntitySerializationStrategy>( v3, v3 );
+
+
+ //now create our set of versions
+ VersionedMigrationSet<MvccEntitySerializationStrategy> set = new VersionedMigrationSet<>( v1Tov3, v2Tov3, current );
+
+ return set;
+
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e96af464/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/CollectionMigrationPlugin.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/CollectionMigrationPlugin.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/CollectionMigrationPlugin.java
index 5713af7..5d52a9f 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/CollectionMigrationPlugin.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/CollectionMigrationPlugin.java
@@ -31,17 +31,24 @@ import org.apache.usergrid.persistence.core.migration.data.newimpls.MigrationDat
import org.apache.usergrid.persistence.core.migration.data.newimpls.MigrationPlugin;
import org.apache.usergrid.persistence.core.migration.data.newimpls.ProgressObserver;
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+
/**
* Migration plugin for the collection module
*/
+@Singleton
public class CollectionMigrationPlugin implements MigrationPlugin {
+ public static final String PLUGIN_NAME = "collections-entity-data";
+
private final DataMigration2<EntityIdScope> entityDataMigration;
private final MigrationDataProvider<EntityIdScope> entityIdScopeDataMigrationProvider;
+ @Inject
public CollectionMigrationPlugin( final DataMigration2<EntityIdScope> entityDataMigration,
final MigrationDataProvider<EntityIdScope> entityIdScopeDataMigrationProvider ) {
this.entityDataMigration = entityDataMigration;
@@ -51,7 +58,7 @@ public class CollectionMigrationPlugin implements MigrationPlugin {
@Override
public String getName() {
- return "collections-entity-data";
+ return PLUGIN_NAME;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e96af464/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/MvccEntityDataMigrationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/MvccEntityDataMigrationImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/MvccEntityDataMigrationImpl.java
index 73efcba..5250b5c 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/MvccEntityDataMigrationImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/MvccEntityDataMigrationImpl.java
@@ -20,21 +20,28 @@
package org.apache.usergrid.persistence.collection.serialization.impl.migration;
-import java.util.Collections;
+import java.util.Iterator;
import java.util.List;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.usergrid.persistence.collection.CollectionScope;
-import org.apache.usergrid.persistence.collection.EntitySet;
import org.apache.usergrid.persistence.collection.MvccEntity;
-import org.apache.usergrid.persistence.collection.mvcc.MvccEntityMigrationStrategy;
import org.apache.usergrid.persistence.collection.serialization.MvccEntitySerializationStrategy;
+import org.apache.usergrid.persistence.collection.serialization.UniqueValue;
+import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
+import org.apache.usergrid.persistence.collection.serialization.impl.UniqueValueImpl;
+import org.apache.usergrid.persistence.collection.util.EntityUtils;
import org.apache.usergrid.persistence.core.migration.data.DataMigrationException;
+import org.apache.usergrid.persistence.core.migration.data.MigrationInfoSerialization;
import org.apache.usergrid.persistence.core.migration.data.newimpls.DataMigration2;
import org.apache.usergrid.persistence.core.migration.data.newimpls.MigrationDataProvider;
+import org.apache.usergrid.persistence.core.migration.data.newimpls.MigrationRelationship;
import org.apache.usergrid.persistence.core.migration.data.newimpls.ProgressObserver;
-import org.apache.usergrid.persistence.core.migration.schema.MigrationStrategy;
+import org.apache.usergrid.persistence.core.migration.data.newimpls.VersionedMigrationSet;
+import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.entity.Id;
+import org.apache.usergrid.persistence.model.field.Field;
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
import com.google.inject.Inject;
@@ -43,6 +50,7 @@ import com.netflix.astyanax.MutationBatch;
import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
import rx.Observable;
+import rx.Subscriber;
import rx.functions.Action1;
import rx.functions.Func1;
import rx.functions.Func2;
@@ -55,20 +63,28 @@ import rx.schedulers.Schedulers;
public class MvccEntityDataMigrationImpl implements DataMigration2<EntityIdScope> {
private final Keyspace keyspace;
- private final MvccEntityMigrationStrategy entityMigrationStrategy;
+ private final VersionedMigrationSet<MvccEntitySerializationStrategy> allVersions;
+ private final UniqueValueSerializationStrategy uniqueValueSerializationStrategy;
+ private final MigrationInfoSerialization migrationInfoSerialization;
@Inject
- public MvccEntityDataMigrationImpl( Keyspace keyspace, MvccEntityMigrationStrategy serializationStrategy ) {
+ public MvccEntityDataMigrationImpl( final Keyspace keyspace,
+ final VersionedMigrationSet<MvccEntitySerializationStrategy> allVersions,
+ final UniqueValueSerializationStrategy uniqueValueSerializationStrategy,
+ final MigrationInfoSerialization migrationInfoSerialization ) {
this.keyspace = keyspace;
- this.entityMigrationStrategy = serializationStrategy;
+ this.allVersions = allVersions;
+ this.uniqueValueSerializationStrategy = uniqueValueSerializationStrategy;
+ this.migrationInfoSerialization = migrationInfoSerialization;
}
@Override
public int getVersion() {
- return entityMigrationStrategy.getVersion();
+ //get the max implementation version, since that's what we're going to
+ return allVersions.getMaxVersion( migrationInfoSerialization.getVersion( CollectionMigrationPlugin.PLUGIN_NAME ) );
}
@@ -77,85 +93,115 @@ public class MvccEntityDataMigrationImpl implements DataMigration2<EntityIdScope
final AtomicLong atomicLong = new AtomicLong();
-
//capture the time the test starts
final UUID startTime = UUIDGenerator.newTimeUUID();
+ final MigrationRelationship<MvccEntitySerializationStrategy>
+ migration = allVersions.getMigrationRelationship( getCurrentSystemVersion() );
+
final long migrated = migrationDataProvider.getData().subscribeOn( Schedulers.io() )
.parallel( new Func1<Observable<EntityIdScope>, Observable<Long>>() {
//process the ids in parallel
@Override
- public Observable<Long> call(
- final Observable<EntityIdScope>
- entityIdScopeObservable ) {
+ public Observable<Long> call( final Observable<EntityIdScope> entityIdScopeObservable ) {
+
- final MutationBatch totalBatch =
- keyspace.prepareMutationBatch();
+ return entityIdScopeObservable
+ .flatMap( new Func1<EntityIdScope, Observable<EntityToSaveMessage>>() {
- return entityIdScopeObservable.doOnNext(
- new Action1<EntityIdScope>() {
- //load the entity and add it to the toal mutation
@Override
- public void call( final EntityIdScope idScope ) {
+ public Observable<EntityToSaveMessage> call(
+ final EntityIdScope entityIdScope ) {
//load the entity
- MigrationStrategy
- .MigrationRelationship<MvccEntitySerializationStrategy>
- migration = entityMigrationStrategy
- .getMigration();
+ final CollectionScope currentScope = entityIdScope.getCollectionScope();
+
+
+
+ //for each element in our history, we need to copy it to v2. Note that this migration
+ //won't support anything beyond V2
+
+ final Iterator<MvccEntity> allVersions = migration.from
+ .loadAscendingHistory( currentScope, entityIdScope.getId(),
+ startTime, 100 );
+
+ //emit all the entities
+ return Observable.create( new Observable.OnSubscribe<EntityToSaveMessage>() {
+ @Override
+ public void call(
+ final Subscriber<? super EntityToSaveMessage> subscriber ) {
+
+ while ( allVersions.hasNext() ) {
+ final EntityToSaveMessage message =
+ new EntityToSaveMessage( currentScope,
+ allVersions.next() );
+ subscriber.onNext( message );
+ }
+
+ subscriber.onCompleted();
+ }
+ } );
+ }
+ } ).buffer( 100 ).doOnNext( new Action1<List<EntityToSaveMessage>>() {
+ @Override
+ public void call( final List<EntityToSaveMessage> messages ) {
+ atomicLong.addAndGet( messages.size() );
+
+ final MutationBatch totalBatch = keyspace.prepareMutationBatch();
+
+
+ for ( EntityToSaveMessage message : messages ) {
+
+ final MutationBatch entityRewrite =
+ migration.to.write( message.scope, message.entity );
+
+ //add to the total batch
+ totalBatch.mergeShallow( entityRewrite );
+
+ //write the unique values
+ if ( message.entity.getEntity().isPresent() ) {
- CollectionScope currentScope =
- idScope.getCollectionScope();
+ final Entity entity = message.entity.getEntity().get();
- //for each element in the history in the
- // previous
- // version,
- // copy it to the CF in v2
+ final Id entityId = entity.getId();
+ final UUID version = message.entity.getVersion();
- EntitySet allVersions = migration.from()
- .load( currentScope,
- Collections
- .singleton(
- idScope.getId() ),
- startTime );
+ // re-write the unique values but this time with no TTL
+ for ( Field field : EntityUtils
+ .getUniqueFields( message.entity.getEntity().get() ) ) {
- final MvccEntity version = allVersions
- .getEntity( idScope.getId() );
+ UniqueValue written =
+ new UniqueValueImpl( field, entityId, version );
- final MutationBatch versionBatch =
- migration.to().write( currentScope,
- version );
+ MutationBatch mb =
+ uniqueValueSerializationStrategy.write( message.scope, written );
- totalBatch.mergeShallow( versionBatch );
+
+ // merge into our existing mutation batch
+ totalBatch.mergeShallow( mb );
+ }
+
+
+ }
+ }
+
+
+ executeBatch( totalBatch, observer, atomicLong );
}
} )
- //every 100 flush the mutation
- .buffer( 100 ).doOnNext(
- new Action1<List<EntityIdScope>>() {
- @Override
- public void call(
- final List<EntityIdScope> ids ) {
- atomicLong.addAndGet( 100 );
- executeBatch( totalBatch, observer,
- atomicLong );
- }
- } )
//count the results
- .reduce( 0l,
- new Func2<Long, List<EntityIdScope>, Long>
- () {
- @Override
- public Long call( final Long aLong,
- final List<EntityIdScope> ids ) {
- return aLong + ids.size();
- }
- } );
+ .reduce( 0l, new Func2<Long, List<EntityToSaveMessage>, Long>() {
+ @Override
+ public Long call( final Long aLong, final List<EntityToSaveMessage> ids ) {
+ return aLong + ids.size();
+ }
+ } );
}
} ).toBlocking().last();
@@ -176,4 +222,19 @@ public class MvccEntityDataMigrationImpl implements DataMigration2<EntityIdScope
throw new DataMigrationException( "Unable to migrate batches ", e );
}
}
+
+ private int getCurrentSystemVersion(){
+ return migrationInfoSerialization.getVersion( CollectionMigrationPlugin.PLUGIN_NAME );
+ }
+
+ private static final class EntityToSaveMessage{
+ private final CollectionScope scope;
+ private final MvccEntity entity;
+
+
+ private EntityToSaveMessage( final CollectionScope scope, final MvccEntity entity ) {
+ this.scope = scope;
+ this.entity = entity;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e96af464/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/util/EntityUtils.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/util/EntityUtils.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/util/EntityUtils.java
index 6233075..beff3bd 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/util/EntityUtils.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/util/EntityUtils.java
@@ -1,13 +1,16 @@
package org.apache.usergrid.persistence.collection.util;
-import java.lang.reflect.Field;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
import java.util.UUID;
import org.apache.commons.lang3.reflect.FieldUtils;
import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
+import org.apache.usergrid.persistence.model.field.Field;
/**
@@ -16,9 +19,9 @@ import org.apache.usergrid.persistence.model.entity.Id;
public class EntityUtils {
- private static final Field VERSION = FieldUtils.getField( Entity.class, "version", true );
+ private static final java.lang.reflect.Field VERSION = FieldUtils.getField( Entity.class, "version", true );
- private static final Field ID = FieldUtils.getField( Entity.class, "id", true );
+ private static final java.lang.reflect.Field ID = FieldUtils.getField( Entity.class, "id", true );
/**
@@ -46,4 +49,24 @@ public class EntityUtils {
throw new RuntimeException( "Unable to set the field " + ID + " into the entity", e );
}
}
+
+
+ /**
+ * Get all unique fields on an entity
+ * @param entity
+ * @return
+ */
+ public static List<Field> getUniqueFields( Entity entity ) {
+ final Collection<Field> entityFields = entity.getFields();
+
+ //preallocate to max possible for more efficient runtime
+ final List<Field> possibleFields = new ArrayList<>( entityFields.size() );
+
+ for ( Field field : entity.getFields() ) {
+ if ( field.isUnique() ) {
+ possibleFields.add( field );
+ }
+ }
+ return possibleFields;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e96af464/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/guice/TestCollectionModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/guice/TestCollectionModule.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/guice/TestCollectionModule.java
index 06f4565..7e1f32b 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/guice/TestCollectionModule.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/guice/TestCollectionModule.java
@@ -21,23 +21,13 @@ package org.apache.usergrid.persistence.collection.guice;
import org.apache.usergrid.persistence.core.guice.CommonModule;
-import org.apache.usergrid.persistence.core.guice.MaxMigrationModule;
import org.apache.usergrid.persistence.core.guice.TestModule;
-import org.apache.usergrid.persistence.core.rx.AllEntitiesInSystemObservable;
-import org.apache.usergrid.persistence.core.rx.AllEntitiesInSystemTestObservable;
-import org.apache.usergrid.persistence.core.rx.ApplicationObservable;
-import org.apache.usergrid.persistence.core.rx.ApplicationsTestObservable;
public class TestCollectionModule extends TestModule {
@Override
protected void configure() {
- /**
- * Runtime modules
- */
- bind(AllEntitiesInSystemObservable.class).to(AllEntitiesInSystemTestObservable.class);
- bind(ApplicationObservable.class).to(ApplicationsTestObservable.class);
install( new CommonModule() );
install( new CollectionModule() );
@@ -45,7 +35,7 @@ public class TestCollectionModule extends TestModule {
/**
* Test modules
*/
- install(new MaxMigrationModule());
+// install(new MaxMigrationModule());
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e96af464/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV1_2Test.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV1_2Test.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV1_2Test.java
deleted file mode 100644
index 6037317..0000000
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV1_2Test.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.persistence.collection.serialization.impl;
-
-
-import org.apache.usergrid.persistence.collection.serialization.MvccEntitySerializationStrategy;
-import org.apache.usergrid.persistence.core.guice.V2Impl;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.runner.RunWith;
-
-import org.apache.usergrid.persistence.collection.guice.TestCollectionModule;
-import org.apache.usergrid.persistence.core.guice.ProxyImpl;
-import org.apache.usergrid.persistence.core.migration.data.MigrationInfoSerialization;
-import org.apache.usergrid.persistence.core.test.ITRunner;
-import org.apache.usergrid.persistence.core.test.UseModules;
-
-import com.google.inject.Inject;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-
-
-@RunWith( ITRunner.class )
-@UseModules( TestCollectionModule.class )
-public class MvccEntitySerializationStrategyProxyV1_2Test extends MvccEntitySerializationStrategyImplTest {
-
- @Inject
- @ProxyImpl
- private MvccEntitySerializationStrategy serializationStrategy;
-
- @Override
- protected MvccEntitySerializationStrategy getMvccEntitySerializationStrategy() {
- return serializationStrategy;
- }
-
- @Inject
- protected MigrationInfoSerialization migrationInfoSerialization;
-
- private int existingVersion;
-
-
- /**
- * We need to run our migration to ensure that we are on the current version, and everything still functions
- * correctly
- */
- @Before
- public void setMigrationVersion() {
- existingVersion = migrationInfoSerialization.getVersion();
-
- //set our version to 0 so it uses both impls of the proxy
- migrationInfoSerialization.setVersion( V2Impl.MIGRATION_VERSION-1 );
- }
-
-
-
-
- @After
- public void reSetMigrationVersion() {
- migrationInfoSerialization.setVersion( existingVersion );
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e96af464/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV1_3Test.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV1_3Test.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV1_3Test.java
index edad4ce..81113b1 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV1_3Test.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV1_3Test.java
@@ -20,38 +20,41 @@
package org.apache.usergrid.persistence.collection.serialization.impl;
-import org.apache.usergrid.persistence.collection.serialization.MvccEntitySerializationStrategy;
-import org.apache.usergrid.persistence.core.guice.V1ProxyImpl;
-import org.apache.usergrid.persistence.core.guice.V2Impl;
import org.junit.After;
import org.junit.Before;
import org.junit.runner.RunWith;
import org.apache.usergrid.persistence.collection.guice.TestCollectionModule;
+import org.apache.usergrid.persistence.collection.serialization.MvccEntitySerializationStrategy;
+import org.apache.usergrid.persistence.collection.serialization.impl.migration.CollectionMigrationPlugin;
+import org.apache.usergrid.persistence.core.guice.ProxyImpl;
import org.apache.usergrid.persistence.core.migration.data.MigrationInfoSerialization;
+import org.apache.usergrid.persistence.core.migration.data.newimpls.VersionedMigrationSet;
import org.apache.usergrid.persistence.core.test.ITRunner;
import org.apache.usergrid.persistence.core.test.UseModules;
import com.google.inject.Inject;
-import static org.junit.Assert.assertTrue;
-
@RunWith( ITRunner.class )
@UseModules( TestCollectionModule.class )
-public class MvccEntitySerializationStrategyProxyV1_3Test extends MvccEntitySerializationStrategyV2Test {
+public class MvccEntitySerializationStrategyProxyV1_3Test extends MvccEntitySerializationStrategyImplTest {
@Inject
- @V1ProxyImpl
+ @ProxyImpl
private MvccEntitySerializationStrategy serializationStrategy;
+ @Inject
+ private MvccEntitySerializationStrategyV1Impl mvccEntitySerializationStrategyV1;
+
+ @Inject
+ private VersionedMigrationSet<MvccEntitySerializationStrategy> migrationSet;
@Override
protected MvccEntitySerializationStrategy getMvccEntitySerializationStrategy() {
return serializationStrategy;
}
-
@Inject
protected MigrationInfoSerialization migrationInfoSerialization;
@@ -64,10 +67,10 @@ public class MvccEntitySerializationStrategyProxyV1_3Test extends MvccEntitySeri
*/
@Before
public void setMigrationVersion() {
- existingVersion = migrationInfoSerialization.getVersion();
+ existingVersion = migrationInfoSerialization.getVersion( CollectionMigrationPlugin.PLUGIN_NAME);
- //set our new version, so that is will run through the new code
- migrationInfoSerialization.setVersion( V2Impl.MIGRATION_VERSION );
+ //set our migration version to be v1
+ migrationInfoSerialization.setVersion( CollectionMigrationPlugin.PLUGIN_NAME, mvccEntitySerializationStrategyV1.getImplementationVersion() );
}
@@ -75,6 +78,7 @@ public class MvccEntitySerializationStrategyProxyV1_3Test extends MvccEntitySeri
@After
public void reSetMigrationVersion() {
- migrationInfoSerialization.setVersion( existingVersion );
+ migrationInfoSerialization.setVersion(CollectionMigrationPlugin.PLUGIN_NAME, existingVersion );
}
+
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e96af464/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV2Test.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV2Test.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV2Test.java
deleted file mode 100644
index 90c34c0..0000000
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV2Test.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.usergrid.persistence.collection.serialization.impl;
-
-import com.google.inject.Inject;
-import org.apache.usergrid.persistence.collection.guice.TestCollectionModule;
-import org.apache.usergrid.persistence.collection.serialization.MvccEntitySerializationStrategy;
-import org.apache.usergrid.persistence.core.guice.ProxyImpl;
-import org.apache.usergrid.persistence.core.guice.V3Impl;
-import org.apache.usergrid.persistence.core.migration.data.MigrationInfoSerialization;
-import org.apache.usergrid.persistence.core.test.ITRunner;
-import org.apache.usergrid.persistence.core.test.UseModules;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.runner.RunWith;
-
-/**
- * Tests migration from 2 -> 3
- */
-
-
-@RunWith( ITRunner.class )
-@UseModules( TestCollectionModule.class )
-public class MvccEntitySerializationStrategyProxyV2Test extends MvccEntitySerializationStrategyV3Test {
-
- @Inject
- @ProxyImpl
- private MvccEntitySerializationStrategy serializationStrategy;
-
-
- @Override
- protected MvccEntitySerializationStrategy getMvccEntitySerializationStrategy() {
- return serializationStrategy;
- }
-
-
- @Inject
- protected MigrationInfoSerialization migrationInfoSerialization;
-
- private int existingVersion;
-
-
- /**
- * We need to run our migration to ensure that we are on the current version, and everything still functions
- * correctly
- */
- @Before
- public void setMigrationVersion() {
- existingVersion = migrationInfoSerialization.getVersion();
-
- //set our new version, so that is will run through the new code
- migrationInfoSerialization.setVersion( V3Impl.MIGRATION_VERSION );
- }
-
-
-
-
- @After
- public void reSetMigrationVersion() {
- migrationInfoSerialization.setVersion( existingVersion );
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e96af464/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV2_3Test.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV2_3Test.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV2_3Test.java
new file mode 100644
index 0000000..da496f7
--- /dev/null
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV2_3Test.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence.collection.serialization.impl;
+
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.runner.RunWith;
+
+import org.apache.usergrid.persistence.collection.guice.TestCollectionModule;
+import org.apache.usergrid.persistence.collection.serialization.MvccEntitySerializationStrategy;
+import org.apache.usergrid.persistence.collection.serialization.impl.migration.CollectionMigrationPlugin;
+import org.apache.usergrid.persistence.core.guice.ProxyImpl;
+import org.apache.usergrid.persistence.core.migration.data.MigrationInfoSerialization;
+import org.apache.usergrid.persistence.core.test.ITRunner;
+import org.apache.usergrid.persistence.core.test.UseModules;
+
+import com.google.inject.Inject;
+
+
+@RunWith( ITRunner.class )
+@UseModules( TestCollectionModule.class )
+public class MvccEntitySerializationStrategyProxyV2_3Test extends MvccEntitySerializationStrategyV2Test {
+
+ @Inject
+ @ProxyImpl
+ private MvccEntitySerializationStrategy serializationStrategy;
+
+ @Inject
+ private MvccEntitySerializationStrategyV2Impl v2Impl;
+
+
+ @Override
+ protected MvccEntitySerializationStrategy getMvccEntitySerializationStrategy() {
+ return serializationStrategy;
+ }
+
+
+ @Inject
+ protected MigrationInfoSerialization migrationInfoSerialization;
+
+ private int existingVersion;
+
+
+ /**
+ * We need to run our migration to ensure that we are on the current version, and everything still functions
+ * correctly
+ */
+ @Before
+ public void setMigrationVersion() {
+ existingVersion = migrationInfoSerialization.getVersion( CollectionMigrationPlugin.PLUGIN_NAME);
+
+ //set our new version, so that is will run through the new code
+ migrationInfoSerialization.setVersion( CollectionMigrationPlugin.PLUGIN_NAME, v2Impl.getImplementationVersion() );
+ }
+
+
+
+
+ @After
+ public void reSetMigrationVersion() {
+ migrationInfoSerialization.setVersion(CollectionMigrationPlugin.PLUGIN_NAME, existingVersion );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e96af464/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV1ImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV1ImplTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV1ImplTest.java
index 1214e0e..d4d6fb5 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV1ImplTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV1ImplTest.java
@@ -20,28 +20,22 @@
package org.apache.usergrid.persistence.collection.serialization.impl;
-import org.apache.usergrid.persistence.collection.serialization.MvccEntitySerializationStrategy;
import org.junit.runner.RunWith;
import org.apache.usergrid.persistence.collection.guice.TestCollectionModule;
-import org.apache.usergrid.persistence.core.guice.V1Impl;
+import org.apache.usergrid.persistence.collection.serialization.MvccEntitySerializationStrategy;
import org.apache.usergrid.persistence.core.test.ITRunner;
import org.apache.usergrid.persistence.core.test.UseModules;
import com.google.inject.Inject;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
@RunWith( ITRunner.class )
@UseModules( TestCollectionModule.class )
public class MvccEntitySerializationStrategyV1ImplTest extends MvccEntitySerializationStrategyImplTest {
@Inject
- @V1Impl
- private MvccEntitySerializationStrategy serializationStrategy;
+ private MvccEntitySerializationStrategyV1Impl serializationStrategy;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e96af464/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2ImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2ImplTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2ImplTest.java
index 6a70273..97bcb1f 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2ImplTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2ImplTest.java
@@ -20,26 +20,22 @@
package org.apache.usergrid.persistence.collection.serialization.impl;
-import org.apache.usergrid.persistence.collection.serialization.MvccEntitySerializationStrategy;
import org.junit.runner.RunWith;
import org.apache.usergrid.persistence.collection.guice.TestCollectionModule;
-import org.apache.usergrid.persistence.core.guice.V2Impl;
+import org.apache.usergrid.persistence.collection.serialization.MvccEntitySerializationStrategy;
import org.apache.usergrid.persistence.core.test.ITRunner;
import org.apache.usergrid.persistence.core.test.UseModules;
import com.google.inject.Inject;
-import static org.junit.Assert.assertTrue;
-
@RunWith( ITRunner.class )
@UseModules( TestCollectionModule.class )
public class MvccEntitySerializationStrategyV2ImplTest extends MvccEntitySerializationStrategyV2Test {
@Inject
- @V2Impl
- private MvccEntitySerializationStrategy serializationStrategy;
+ private MvccEntitySerializationStrategyV2Impl serializationStrategy;
@Override
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e96af464/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3ImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3ImplTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3ImplTest.java
index 8ace3ff..b30e901 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3ImplTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3ImplTest.java
@@ -10,7 +10,6 @@ import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.collection.guice.TestCollectionModule;
import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl;
import org.apache.usergrid.persistence.collection.serialization.MvccEntitySerializationStrategy;
-import org.apache.usergrid.persistence.core.guice.V3Impl;
import org.apache.usergrid.persistence.core.test.ITRunner;
import org.apache.usergrid.persistence.core.test.UseModules;
import org.apache.usergrid.persistence.model.entity.Id;
@@ -30,8 +29,7 @@ import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
@UseModules( TestCollectionModule.class )
public class MvccEntitySerializationStrategyV3ImplTest extends MvccEntitySerializationStrategyV3Test {
@Inject
- @V3Impl
- private MvccEntitySerializationStrategy serializationStrategy;
+ private MvccEntitySerializationStrategyV3Impl serializationStrategy;
@Override
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e96af464/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/guice/V1Impl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/guice/V1Impl.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/guice/V1Impl.java
deleted file mode 100644
index 4fba781..0000000
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/guice/V1Impl.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.usergrid.persistence.core.guice;
-
-import com.google.inject.BindingAnnotation;
-
-import java.lang.annotation.Retention;
-import java.lang.annotation.Target;
-
-import static java.lang.annotation.ElementType.FIELD;
-import static java.lang.annotation.ElementType.METHOD;
-import static java.lang.annotation.ElementType.PARAMETER;
-import static java.lang.annotation.RetentionPolicy.RUNTIME;
-
-/**
- * Represents 2 versions of an impl. Generally used for online migration. This represents the version that is the
- * current version of the implementation. I.E the "new" version.
- */
-@BindingAnnotation
-@Target({ FIELD, PARAMETER, METHOD }) @Retention(RUNTIME)
-public @interface V1Impl {}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e96af464/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/guice/V1ProxyImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/guice/V1ProxyImpl.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/guice/V1ProxyImpl.java
deleted file mode 100644
index e1e7741..0000000
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/guice/V1ProxyImpl.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.usergrid.persistence.core.guice;
-
-import com.google.inject.BindingAnnotation;
-
-import java.lang.annotation.Retention;
-import java.lang.annotation.Target;
-
-import static java.lang.annotation.ElementType.FIELD;
-import static java.lang.annotation.ElementType.METHOD;
-import static java.lang.annotation.ElementType.PARAMETER;
-import static java.lang.annotation.RetentionPolicy.RUNTIME;
-
-/**
- * V1 proxy annotation
- */
-@BindingAnnotation
-@Target({ FIELD, PARAMETER, METHOD }) @Retention(RUNTIME)
-public @interface V1ProxyImpl {
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e96af464/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/guice/V2Impl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/guice/V2Impl.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/guice/V2Impl.java
deleted file mode 100644
index 0ee87f0..0000000
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/guice/V2Impl.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.usergrid.persistence.core.guice;
-
-import com.google.inject.BindingAnnotation;
-
-import java.lang.annotation.Retention;
-import java.lang.annotation.Target;
-
-import static java.lang.annotation.ElementType.FIELD;
-import static java.lang.annotation.ElementType.METHOD;
-import static java.lang.annotation.ElementType.PARAMETER;
-import static java.lang.annotation.RetentionPolicy.RUNTIME;
-
-/**
- * Represents 2 versions of an impl. Generally used for online migration. This represents the version that is the
- * current version of the implementation. I.E the "new" version.
- */
-@BindingAnnotation
-@Target({ FIELD, PARAMETER, METHOD }) @Retention(RUNTIME)
-public @interface V2Impl {
- public static final int MIGRATION_VERSION = 3;
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e96af464/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/guice/V3Impl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/guice/V3Impl.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/guice/V3Impl.java
deleted file mode 100644
index aef3a9a..0000000
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/guice/V3Impl.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.usergrid.persistence.core.guice;
-
-import com.google.inject.BindingAnnotation;
-
-import java.lang.annotation.Retention;
-import java.lang.annotation.Target;
-
-import static java.lang.annotation.ElementType.FIELD;
-import static java.lang.annotation.ElementType.METHOD;
-import static java.lang.annotation.ElementType.PARAMETER;
-import static java.lang.annotation.RetentionPolicy.RUNTIME;
-
-/**
- * Represents 2 versions of an impl. Generally used for online migration. This represents the version that is the
- * current version of the implementation. I.E the "new" version.
- */
-@BindingAnnotation
-@Target({ FIELD, PARAMETER, METHOD }) @Retention(RUNTIME)
-public @interface V3Impl {
- public static final int MIGRATION_VERSION = 4;
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e96af464/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/ApplicationDataMigration.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/ApplicationDataMigration.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/ApplicationDataMigration.java
deleted file mode 100644
index bbff0b9..0000000
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/ApplicationDataMigration.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- *
- * * Licensed to the Apache Software Foundation (ASF) under one or more
- * * contributor license agreements. The ASF licenses this file to You
- * * under the Apache License, Version 2.0 (the "License"); you may not
- * * use this file except in compliance with the License.
- * * You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing, software
- * * distributed under the License is distributed on an "AS IS" BASIS,
- * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * * See the License for the specific language governing permissions and
- * * limitations under the License. For additional information regarding
- * * copyright in this work, please see the NOTICE file in the top level
- * * directory of this distribution.
- *
- */
-package org.apache.usergrid.persistence.core.migration.data;
-
-import org.apache.usergrid.persistence.core.scope.ApplicationEntityGroup;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-
-
-/**
- * Migrate applications
- */
-public interface ApplicationDataMigration extends DataMigration<ApplicationScope> {
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e96af464/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/CollectionDataMigration.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/CollectionDataMigration.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/CollectionDataMigration.java
deleted file mode 100644
index df1af72..0000000
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/CollectionDataMigration.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- *
- * * Licensed to the Apache Software Foundation (ASF) under one or more
- * * contributor license agreements. The ASF licenses this file to You
- * * under the Apache License, Version 2.0 (the "License"); you may not
- * * use this file except in compliance with the License.
- * * You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing, software
- * * distributed under the License is distributed on an "AS IS" BASIS,
- * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * * See the License for the specific language governing permissions and
- * * limitations under the License. For additional information regarding
- * * copyright in this work, please see the NOTICE file in the top level
- * * directory of this distribution.
- *
- */
-package org.apache.usergrid.persistence.core.migration.data;
-
-import org.apache.usergrid.persistence.core.scope.ApplicationEntityGroup;
-
-
-/**
- * Migrate Collections
- */
-public interface CollectionDataMigration extends DataMigration<ApplicationEntityGroup> {
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e96af464/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManager.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManager.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManager.java
index 1ca8d90..7ddc6e1 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManager.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManager.java
@@ -54,11 +54,6 @@ public interface DataMigrationManager {
*/
public void resetToVersion(final String pluginName, final int version);
- /**
- * Invalidate the cache for versions
- */
- public void invalidate();
-
/**
* Return that last status of the migration
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e96af464/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManagerImpl.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManagerImpl.java
index ddb0a61..41e76e8 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManagerImpl.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManagerImpl.java
@@ -51,18 +51,6 @@ public class DataMigrationManagerImpl implements DataMigrationManager {
private final MigrationInfoSerialization migrationInfoSerialization;
- /**
- * Cache to cache versions temporarily
- */
- private final LoadingCache<String, Integer> versionCache = CacheBuilder.newBuilder()
- //cache the local value for 1 minute
- .expireAfterWrite( 1, TimeUnit.MINUTES ).build( new CacheLoader<String, Integer>() {
- @Override
- public Integer load( final String key ) throws Exception {
- return migrationInfoSerialization.getVersion( key );
- }
- } );
-
@Inject
public DataMigrationManagerImpl( final Set<MigrationPlugin> plugins,
@@ -121,11 +109,6 @@ public class DataMigrationManagerImpl implements DataMigrationManager {
}
- @Override
- public void invalidate() {
- versionCache.invalidateAll();
- }
-
@Override
public int getCurrentVersion( final String pluginName ) {
[14/50] [abbrv] incubator-usergrid git commit: Tests for migrations
in both modules complete
Posted by sf...@apache.org.
Tests for migrations in both modules complete
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/48127152
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/48127152
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/48127152
Branch: refs/heads/USERGRID-480
Commit: 481271523b0056f7cf9bf21194e166f490fabb09
Parents: 802dcde
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Mar 4 11:43:13 2015 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Mar 4 11:43:13 2015 -0700
----------------------------------------------------------------------
.../collection/impl/EntityDeletedTask.java | 8 +-
.../migration/MvccEntityDataMigrationImpl.java | 405 +++++++------------
...ctMvccEntityDataMigrationV1ToV3ImplTest.java | 217 ++++++++++
.../MvccEntityDataMigrationV1ToV3ImplTest.java | 106 ++---
.../MvccEntityDataMigrationV2ToV3ImplTest.java | 100 +++++
.../core/guice/MigrationManagerRule.java | 7 +
6 files changed, 498 insertions(+), 345 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/48127152/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityDeletedTask.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityDeletedTask.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityDeletedTask.java
index 0b4b739..dbe58d4 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityDeletedTask.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityDeletedTask.java
@@ -107,12 +107,10 @@ public class EntityDeletedTask implements Task<Void> {
final MutationBatch entityDelete = entitySerializationStrategy.delete(collectionScope, entityId, version);
final MutationBatch logDelete = logEntrySerializationStrategy.delete(collectionScope, entityId, version);
- throw new NotImplementedException( "Implement unique cleanup here" );
+ entityDelete.execute();
+ logDelete.execute();
//
-// entityDelete.execute();
-// logDelete.execute();
-//
-// return null;
+ return null;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/48127152/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/MvccEntityDataMigrationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/MvccEntityDataMigrationImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/MvccEntityDataMigrationImpl.java
index 7580a26..05e5db5 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/MvccEntityDataMigrationImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/MvccEntityDataMigrationImpl.java
@@ -20,24 +20,27 @@
package org.apache.usergrid.persistence.collection.serialization.impl.migration;
+import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicLong;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.collection.EntityVersionCleanupFactory;
import org.apache.usergrid.persistence.collection.MvccEntity;
+import org.apache.usergrid.persistence.collection.impl.EntityDeletedTask;
import org.apache.usergrid.persistence.collection.impl.EntityVersionCleanupTask;
import org.apache.usergrid.persistence.collection.serialization.MvccEntitySerializationStrategy;
import org.apache.usergrid.persistence.collection.serialization.UniqueValue;
import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
-import org.apache.usergrid.persistence.collection.serialization.impl.MvccEntitySerializationStrategyImpl;
import org.apache.usergrid.persistence.collection.serialization.impl.MvccEntitySerializationStrategyV3Impl;
import org.apache.usergrid.persistence.collection.serialization.impl.UniqueValueImpl;
import org.apache.usergrid.persistence.collection.util.EntityUtils;
import org.apache.usergrid.persistence.core.migration.data.DataMigrationException;
-import org.apache.usergrid.persistence.core.migration.data.MigrationInfoSerialization;
import org.apache.usergrid.persistence.core.migration.data.newimpls.DataMigration2;
import org.apache.usergrid.persistence.core.migration.data.newimpls.MigrationDataProvider;
import org.apache.usergrid.persistence.core.migration.data.newimpls.MigrationRelationship;
@@ -56,11 +59,8 @@ import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
import rx.Observable;
import rx.Subscriber;
-import rx.functions.Action0;
import rx.functions.Action1;
import rx.functions.Func1;
-import rx.functions.Func2;
-import rx.observables.GroupedObservable;
import rx.schedulers.Schedulers;
@@ -70,6 +70,9 @@ import rx.schedulers.Schedulers;
@Singleton
public class MvccEntityDataMigrationImpl implements DataMigration2<EntityIdScope> {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger( MvccEntityDataMigrationImpl.class );
+
private final Keyspace keyspace;
private final VersionedMigrationSet<MvccEntitySerializationStrategy> allVersions;
private final UniqueValueSerializationStrategy uniqueValueSerializationStrategy;
@@ -77,12 +80,14 @@ public class MvccEntityDataMigrationImpl implements DataMigration2<EntityIdScope
private final MvccEntitySerializationStrategyV3Impl mvccEntitySerializationStrategyV3;
+
@Inject
public MvccEntityDataMigrationImpl( final Keyspace keyspace,
final VersionedMigrationSet<MvccEntitySerializationStrategy> allVersions,
final UniqueValueSerializationStrategy uniqueValueSerializationStrategy,
final EntityVersionCleanupFactory entityVersionCleanupFactory,
- final MvccEntitySerializationStrategyV3Impl mvccEntitySerializationStrategyV3 ) {
+ final MvccEntitySerializationStrategyV3Impl mvccEntitySerializationStrategyV3
+ ) {
this.keyspace = keyspace;
this.allVersions = allVersions;
@@ -107,7 +112,7 @@ public class MvccEntityDataMigrationImpl implements DataMigration2<EntityIdScope
@Override
public int migrate( final int currentVersion, final MigrationDataProvider<EntityIdScope> migrationDataProvider,
- final ProgressObserver observer ) {
+ final ProgressObserver observer ) {
final AtomicLong atomicLong = new AtomicLong();
@@ -116,264 +121,147 @@ public class MvccEntityDataMigrationImpl implements DataMigration2<EntityIdScope
final UUID startTime = UUIDGenerator.newTimeUUID();
final MigrationRelationship<MvccEntitySerializationStrategy> migration =
- allVersions.getMigrationRelationship( currentVersion );
-
-
- final long migrated = migrationDataProvider.getData().subscribeOn( Schedulers.io() )
- .parallel( new Func1<Observable<EntityIdScope>, Observable<Long>>() {
-
-
- //process the ids in parallel
- @Override
- public Observable<Long> call(
- final Observable<EntityIdScope>
- entityIdScopeObservable ) {
-
-
- return entityIdScopeObservable.flatMap(
- new Func1<EntityIdScope,
- Observable<EntityToSaveMessage>>() {
-
-
- @Override
- public
- Observable<EntityToSaveMessage> call(
- final EntityIdScope
- entityIdScope ) {
-
- //load the entity
- final CollectionScope
- currentScope =
- entityIdScope
- .getCollectionScope();
-
-
- //for each element in our
- // history, we need to copy it
- // to v2.
- // Note that
- // this migration
- //won't support anything beyond V2
-
- final Iterator<MvccEntity>
- allVersions =
- migration.from
- .loadAscendingHistory(
- currentScope,
- entityIdScope
- .getId(),
- startTime,
- 100 );
-
- //emit all the entity versions
- return Observable.create(
- new Observable
- .OnSubscribe<EntityToSaveMessage>() {
- @Override
- public void call(
- final
- Subscriber<? super
- EntityToSaveMessage> subscriber ) {
-
- while ( allVersions
- .hasNext() ) {
- final
- EntityToSaveMessage
- message =
- new EntityToSaveMessage(
- currentScope,
- allVersions
- .next() );
- subscriber.onNext( message );
- }
-
- subscriber.onCompleted();
- }
- } );
- }
- } )
-
-
- //group them by entity id so we can get
- // the max for cleanup
- .groupBy(
- new Func1<EntityToSaveMessage,
- Id>() {
- @Override
- public Id call(
- final
- EntityToSaveMessage
- entityToSaveMessage ) {
- return entityToSaveMessage.entity
- .getId();
- }
- } )
- //buffer up 10 of groups so we can put them all in a single mutation
- .buffer( 10 ).doOnNext(
- new Action1<List<GroupedObservable<Id, EntityToSaveMessage>>>() {
-
-
- @Override
- public void call(
- final
- List<GroupedObservable<Id, EntityToSaveMessage>> groupedObservables ) {
-
- atomicLong.addAndGet(
- groupedObservables
- .size() );
-
- final MutationBatch
- totalBatch =
- keyspace.prepareMutationBatch();
-
-
- //run each of the
- // groups and add
- // it ot the batch
- Observable
- .from( groupedObservables )
- //emit the group as an observable
- .flatMap(
- new Func1<GroupedObservable<Id, EntityToSaveMessage>, Observable<EntityToSaveMessage>>() {
-
-
- @Override
- public Observable<EntityToSaveMessage> call(
- final GroupedObservable<Id, EntityToSaveMessage> idEntityToSaveMessageGroupedObservable ) {
- return idEntityToSaveMessageGroupedObservable
- .asObservable();
- }
- } )
-
- //merge and add the batch
- .doOnNext(
- new Action1<EntityToSaveMessage>() {
- @Override
- public void call(
- final EntityToSaveMessage message ) {
-
- final MutationBatch
- entityRewrite =
- migration.to.write( message.scope,
- message.entity );
-
- //add to
- // the
- // total
- // batch
- totalBatch.mergeShallow( entityRewrite );
-
- //write
- // the
- // unique values
-
- if ( !message.entity
- .getEntity()
- .isPresent() ) {
- return;
- }
-
- final Entity
- entity =
- message.entity
- .getEntity()
- .get();
-
- final Id
- entityId =
- entity.getId();
-
- final UUID
- version =
- message.entity
- .getVersion();
-
- // re-write the unique values
- // but this
- // time with
- // no TTL so that cleanup can clean up older values
- for ( Field field : EntityUtils
- .getUniqueFields(
- message.entity
- .getEntity()
- .get() ) ) {
-
- UniqueValue
- written =
- new UniqueValueImpl(
- field,
- entityId,
- version );
-
- MutationBatch
- mb =
- uniqueValueSerializationStrategy
- .write( message.scope,
- written );
-
-
- // merge into our
- // existing mutation
- // batch
- totalBatch
- .mergeShallow(
- mb );
- }
- }
- } )
- //once we've streamed everything, flush it
- .doOnCompleted(
-
- new Action0() {
- @Override
- public void call() {
-
- executeBatch(migration.to.getImplementationVersion(), totalBatch, observer, atomicLong );
- }
- } )
- .toBlocking()
- .last();
- }
- } ).
- reduce( 0l,
- new Func2<Long, List<GroupedObservable<Id, EntityToSaveMessage>>, Long>() {
-
- @Override
- public Long call(
- final Long aLong,
- final List<GroupedObservable<Id, EntityToSaveMessage>> groupedObservables ) {
-
- long newCount =
- aLong;
-
- for ( GroupedObservable<Id, EntityToSaveMessage> group : groupedObservables ) {
- newCount +=
- group.longCount()
- .toBlocking()
- .last();
- }
-
- return newCount;
- }
- }
-
-
- );
- }}).toBlocking().last();
-
- //now we update the progress observer
-
- observer.update( migration.to.getImplementationVersion(), "Finished for this step. Migrated " + migrated + "entities total. " );
+ allVersions.getMigrationRelationship( currentVersion );
+
+
+ final Observable<List<EntityToSaveMessage>> migrated =
+ migrationDataProvider.getData().subscribeOn( Schedulers.io() ).parallel(
+ new Func1<Observable<EntityIdScope>, Observable<List<EntityToSaveMessage>>>() {
+
+
+ //process the ids in parallel
+ @Override
+ public Observable<List<EntityToSaveMessage>> call(
+ final Observable<EntityIdScope> entityIdScopeObservable ) {
+
+
+ return entityIdScopeObservable.flatMap(
+ new Func1<EntityIdScope, Observable<EntityToSaveMessage>>() {
+
+
+ @Override
+ public Observable<EntityToSaveMessage> call( final EntityIdScope entityIdScope ) {
+
+ //load the entity
+ final CollectionScope currentScope = entityIdScope.getCollectionScope();
+
+
+ //for each element in our
+ // history, we need to copy it
+ // to v2.
+ // Note that
+ // this migration
+ //won't support anything beyond V2
+
+ final Iterator<MvccEntity> allVersions = migration.from
+ .loadAscendingHistory( currentScope, entityIdScope.getId(), startTime, 100 );
+
+ //emit all the entity versions
+ return Observable.create( new Observable.OnSubscribe<EntityToSaveMessage>() {
+ @Override
+ public void call( final Subscriber<? super
+ EntityToSaveMessage> subscriber ) {
+
+ while ( allVersions.hasNext() ) {
+ final EntityToSaveMessage message = new EntityToSaveMessage( currentScope, allVersions.next() );
+ subscriber.onNext( message );
+ }
+
+ subscriber.onCompleted();
+ }
+ } );
+ }
+ } )
+ //buffer 10 versions
+ .buffer( 100 ).doOnNext( new Action1<List<EntityToSaveMessage>>() {
+ @Override
+ public void call( final List<EntityToSaveMessage> entities ) {
+
+ final MutationBatch totalBatch = keyspace.prepareMutationBatch();
+
+ atomicLong.addAndGet( entities.size() );
+
+ List<EntityVersionCleanupTask> entityVersionCleanupTasks = new ArrayList(entities.size());
+
+ for ( EntityToSaveMessage message : entities ) {
+ final MutationBatch entityRewrite =
+ migration.to.write( message.scope, message.entity );
+
+ //add to
+ // the
+ // total
+ // batch
+ totalBatch.mergeShallow( entityRewrite );
+
+ //write
+ // the
+ // unique values
+
+ if ( !message.entity.getEntity().isPresent() ) {
+ return;
+ }
+
+ final Entity entity = message.entity.getEntity().get();
+
+ final Id entityId = entity.getId();
+
+ final UUID version = message.entity.getVersion();
+
+ // re-write the unique
+ // values
+ // but this
+ // time with
+ // no TTL so that cleanup can clean up
+ // older values
+ for ( Field field : EntityUtils
+ .getUniqueFields( message.entity.getEntity().get() ) ) {
+
+ UniqueValue written = new UniqueValueImpl( field, entityId, version );
+
+ MutationBatch mb =
+ uniqueValueSerializationStrategy.write( message.scope, written );
+
+
+ // merge into our
+ // existing mutation
+ // batch
+ totalBatch.mergeShallow( mb );
+ }
+
+ final EntityVersionCleanupTask task = entityVersionCleanupFactory.getTask( message.scope, message.entity.getId(), version );
+
+ entityVersionCleanupTasks.add( task );
+ }
+
+ executeBatch( migration.to.getImplementationVersion(), totalBatch, observer, atomicLong );
+
+ //now run our cleanup task
+
+ for(EntityVersionCleanupTask entityVersionCleanupTask: entityVersionCleanupTasks){
+ try {
+ entityVersionCleanupTask.call();
+ }
+ catch ( Exception e ) {
+ LOGGER.error( "Unable to run cleanup task", e );
+ }
+ }
+ }
+ } );
+ }
+ } );
+
+ migrated.toBlocking().lastOrDefault(null);
return migration.to.getImplementationVersion();
}
- protected void executeBatch( final int targetVersion, final MutationBatch batch, final ProgressObserver po, final AtomicLong count ) {
+ protected void executeBatch( final int targetVersion, final MutationBatch batch, final ProgressObserver po,
+ final AtomicLong count ) {
try {
batch.execute();
- po.update(targetVersion, "Finished copying " + count + " entities to the new format" );
+ po.update( targetVersion, "Finished copying " + count + " entities to the new format" );
}
catch ( ConnectionException e ) {
po.failed( targetVersion, "Failed to execute mutation in cassandra" );
@@ -382,8 +270,6 @@ public class MvccEntityDataMigrationImpl implements DataMigration2<EntityIdScope
}
-
-
private static final class EntityToSaveMessage {
private final CollectionScope scope;
private final MvccEntity entity;
@@ -394,4 +280,5 @@ public class MvccEntityDataMigrationImpl implements DataMigration2<EntityIdScope
this.entity = entity;
}
}
+
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/48127152/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/AbstractMvccEntityDataMigrationV1ToV3ImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/AbstractMvccEntityDataMigrationV1ToV3ImplTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/AbstractMvccEntityDataMigrationV1ToV3ImplTest.java
new file mode 100644
index 0000000..9894f07
--- /dev/null
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/AbstractMvccEntityDataMigrationV1ToV3ImplTest.java
@@ -0,0 +1,217 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence.collection.serialization.impl.migration;
+
+
+import java.util.UUID;
+
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+import org.apache.usergrid.persistence.collection.CollectionScope;
+import org.apache.usergrid.persistence.collection.MvccEntity;
+import org.apache.usergrid.persistence.collection.guice.TestCollectionModule;
+import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl;
+import org.apache.usergrid.persistence.collection.mvcc.entity.impl.MvccEntityImpl;
+import org.apache.usergrid.persistence.collection.serialization.MvccEntitySerializationStrategy;
+import org.apache.usergrid.persistence.collection.serialization.impl.CollectionDataVersions;
+import org.apache.usergrid.persistence.collection.serialization.impl.MvccEntitySerializationStrategyV1Impl;
+import org.apache.usergrid.persistence.collection.serialization.impl.MvccEntitySerializationStrategyV3Impl;
+import org.apache.usergrid.persistence.core.guice.DataMigrationResetRule;
+import org.apache.usergrid.persistence.core.guice.MigrationManagerRule;
+import org.apache.usergrid.persistence.core.migration.data.DataMigrationManager;
+import org.apache.usergrid.persistence.core.migration.data.TestProgressObserver;
+import org.apache.usergrid.persistence.core.migration.data.newimpls.MigrationDataProvider;
+import org.apache.usergrid.persistence.core.migration.data.newimpls.MigrationRelationship;
+import org.apache.usergrid.persistence.core.migration.data.newimpls.VersionedMigrationSet;
+import org.apache.usergrid.persistence.core.test.ITRunner;
+import org.apache.usergrid.persistence.core.test.UseModules;
+import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.entity.Id;
+import org.apache.usergrid.persistence.model.entity.SimpleId;
+import org.apache.usergrid.persistence.model.util.UUIDGenerator;
+
+import com.google.inject.Inject;
+import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
+
+import net.jcip.annotations.NotThreadSafe;
+
+import rx.Observable;
+
+import static org.apache.usergrid.persistence.core.util.IdGenerator.createId;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+
+@NotThreadSafe
+@RunWith( ITRunner.class )
+@UseModules( { TestCollectionModule.class } )
+public abstract class AbstractMvccEntityDataMigrationV1ToV3ImplTest implements DataMigrationResetRule.DataMigrationManagerProvider {
+
+
+ @Inject
+ @Rule
+ public MigrationManagerRule migrationManagerRule;
+
+
+ @Inject
+ public DataMigrationManager dataMigrationManager;
+
+
+ @Inject
+ public MvccEntityDataMigrationImpl mvccEntityDataMigrationImpl;
+
+
+ @Inject
+ public VersionedMigrationSet<MvccEntitySerializationStrategy> versions;
+
+ /**
+ * Rule to do the resets we need
+ */
+ @Rule
+ public DataMigrationResetRule migrationTestRule =
+ new DataMigrationResetRule( this, CollectionMigrationPlugin.PLUGIN_NAME,
+ CollectionDataVersions.INITIAL.getVersion() );
+
+
+ @Test
+ public void testMigration() throws ConnectionException {
+
+ final Id applicationId = createId( "application" );
+ final String collectionName = "things";
+
+ CollectionScope scope = new CollectionScopeImpl( applicationId, applicationId, collectionName );
+
+ final MvccEntity entity1 = getEntity( "thing" );
+ final MvccEntity entity2 = getEntity( "thing" );
+
+
+ MvccEntitySerializationStrategy v1Impl = getExpectedSourceImpl();
+
+ MvccEntitySerializationStrategy v3Impl = getExpectedTargetImpl();
+
+
+ v1Impl.write( scope, entity1 ).execute();
+ v1Impl.write( scope, entity2 ).execute();
+
+
+ MvccEntity returned1 = v1Impl.load( scope, entity1.getId() ).get();
+ MvccEntity returned2 = v1Impl.load( scope, entity2.getId() ).get();
+
+ assertEquals( "Same entity", entity1, returned1 );
+ assertEquals( "Same entity", entity2, returned2 );
+
+ final Observable<EntityIdScope> entityIdScope =
+ Observable.from( new EntityIdScope( scope, entity1.getId() ), new EntityIdScope( scope, entity2.getId() ) );
+
+
+ final MigrationDataProvider<EntityIdScope> migrationProvider = new MigrationDataProvider<EntityIdScope>() {
+ @Override
+ public Observable<EntityIdScope> getData() {
+ return entityIdScope;
+ }
+ };
+
+ final TestProgressObserver progressObserver = new TestProgressObserver();
+
+ final CollectionDataVersions startVersion = getSourceVersion();
+
+ final MigrationRelationship<MvccEntitySerializationStrategy> tuple =
+ versions.getMigrationRelationship( startVersion.getVersion() );
+
+
+ assertEquals( "Same instance for from", v1Impl.getClass(), tuple.from.getClass() );
+ assertEquals( "Same instance for to", v3Impl.getClass(), tuple.to.getClass() );
+
+ //now migration
+ final int newVersion = mvccEntityDataMigrationImpl
+ .migrate( startVersion.getVersion(), migrationProvider, progressObserver );
+
+
+ final CollectionDataVersions expectedVersion = expectedTargetVersion();
+
+ assertEquals( "Correct version returned", newVersion, expectedVersion.getVersion() );
+ assertFalse( "Progress observer should not have failed", progressObserver.getFailed() );
+ assertTrue( "Progress observer should have update messages", progressObserver.getUpdates().size() > 0 );
+
+
+ //now verify we can read the data correctly in the new version
+ returned1 = v3Impl.load( scope, entity1.getId() ).get();
+ returned2 = v3Impl.load( scope, entity2.getId() ).get();
+
+ assertEquals( "Same entity", entity1, returned1 );
+ assertEquals( "Same entity", entity2, returned2 );
+
+ //verify the tuple is correct
+
+ final MigrationRelationship<MvccEntitySerializationStrategy> newTuple =
+ versions.getMigrationRelationship( newVersion );
+
+
+ assertEquals( "Same instance for from", v3Impl.getClass(), newTuple.from.getClass() );
+ assertEquals( "Same instance for to", v3Impl.getClass(), newTuple.to.getClass() );
+ }
+
+
+ private MvccEntity getEntity( final String type ) {
+
+ final SimpleId entityId = new SimpleId( type );
+ final UUID version = UUIDGenerator.newTimeUUID();
+ final Entity entity = new Entity( entityId );
+
+ MvccEntityImpl logEntry = new MvccEntityImpl( entityId, version, MvccEntity.Status.COMPLETE, entity );
+
+
+ return logEntry;
+ }
+
+
+ @Override
+ public DataMigrationManager getDataMigrationManager() {
+ return dataMigrationManager;
+ }
+
+
+ /**
+ * Get the expected source mvcc implementation for this test
+ * @return
+ */
+ protected abstract MvccEntitySerializationStrategy getExpectedSourceImpl();
+
+ /**
+ * Get the expected target mvcc for this test
+ * @return
+ */
+ protected abstract MvccEntitySerializationStrategy getExpectedTargetImpl();
+
+ /**
+ * Get the expected start version
+ * @return
+ */
+ protected abstract CollectionDataVersions getSourceVersion();
+
+ /**
+ *
+ * @return
+ */
+ protected abstract CollectionDataVersions expectedTargetVersion();
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/48127152/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/MvccEntityDataMigrationV1ToV3ImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/MvccEntityDataMigrationV1ToV3ImplTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/MvccEntityDataMigrationV1ToV3ImplTest.java
index dde67cb..f333fba 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/MvccEntityDataMigrationV1ToV3ImplTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/MvccEntityDataMigrationV1ToV3ImplTest.java
@@ -36,6 +36,7 @@ import org.apache.usergrid.persistence.collection.serialization.impl.CollectionD
import org.apache.usergrid.persistence.collection.serialization.impl.MvccEntitySerializationStrategyV1Impl;
import org.apache.usergrid.persistence.collection.serialization.impl.MvccEntitySerializationStrategyV3Impl;
import org.apache.usergrid.persistence.core.guice.DataMigrationResetRule;
+import org.apache.usergrid.persistence.core.guice.MigrationManagerRule;
import org.apache.usergrid.persistence.core.migration.data.DataMigrationManager;
import org.apache.usergrid.persistence.core.migration.data.TestProgressObserver;
import org.apache.usergrid.persistence.core.migration.data.newimpls.MigrationDataProvider;
@@ -65,7 +66,13 @@ import static org.junit.Assert.assertTrue;
@NotThreadSafe
@RunWith( ITRunner.class )
@UseModules( { TestCollectionModule.class } )
-public class MvccEntityDataMigrationV1ToV3ImplTest implements DataMigrationResetRule.DataMigrationManagerProvider {
+public class MvccEntityDataMigrationV1ToV3ImplTest extends AbstractMvccEntityDataMigrationV1ToV3ImplTest{
+
+
+ @Inject
+ @Rule
+ public MigrationManagerRule migrationManagerRule;
+
@Inject
public DataMigrationManager dataMigrationManager;
@@ -79,101 +86,38 @@ public class MvccEntityDataMigrationV1ToV3ImplTest implements DataMigrationReset
@Inject
public MvccEntityDataMigrationImpl mvccEntityDataMigrationImpl;
+
@Inject
public VersionedMigrationSet<MvccEntitySerializationStrategy> versions;
- /**
- * Rule to do the resets we need
- */
- @Rule
- public DataMigrationResetRule migrationTestRule =
- new DataMigrationResetRule( this, CollectionMigrationPlugin.PLUGIN_NAME,
- CollectionDataVersions.INITIAL.getVersion() );
-
-
- @Test
- public void testMigration() throws ConnectionException {
-
- final Id applicationId = createId("application");
- final String collectionName = "things";
-
- CollectionScope scope = new CollectionScopeImpl(applicationId, applicationId, collectionName );
-
- final MvccEntity entity1 = getEntity( "thing" );
- final MvccEntity entity2 = getEntity( "thing" );
-
- v1Impl.write( scope, entity1 ).execute();
- v1Impl.write( scope, entity2 ).execute();
-
-
- MvccEntity returned1 = v1Impl.load( scope, entity1.getId() ).get();
- MvccEntity returned2 = v1Impl.load( scope, entity2.getId() ).get();
-
- assertEquals("Same entity", entity1, returned1);
- assertEquals("Same entity", entity2, returned2);
-
- final Observable<EntityIdScope> entityIdScope = Observable.from( new EntityIdScope( scope, entity1.getId() ), new EntityIdScope( scope, entity2.getId() ) );
-
-
- final MigrationDataProvider<EntityIdScope> migrationProvider = new MigrationDataProvider<EntityIdScope>() {
- @Override
- public Observable<EntityIdScope> getData() {
- return entityIdScope;
- }
- };
-
- final TestProgressObserver progressObserver = new TestProgressObserver();
-
- //now migration
- final int newVersion = mvccEntityDataMigrationImpl.migrate( CollectionDataVersions.INITIAL.getVersion(), migrationProvider, progressObserver );
-
-
- assertEquals( "Correct version returned", newVersion, CollectionDataVersions.LOG_REMOVAL.getVersion() );
- assertFalse( "Progress observer should not have failed", progressObserver.getFailed() );
- assertTrue( "Progress observer should have update messages", progressObserver.getUpdates().size() > 0 );
-
-
- //now verify we can read the data correctly in the new version
- returned1 = v3Impl.load( scope, entity1.getId() ).get();
- returned2 = v3Impl.load( scope, entity2.getId() ).get();
-
- assertEquals("Same entity", entity1, returned1);
- assertEquals("Same entity", entity2, returned2);
-
- //verify the tuple is correct
-
- final MigrationRelationship<MvccEntitySerializationStrategy>
- tuple = versions.getMigrationRelationship( newVersion );
-
-
- assertSame("Same instance for from", v1Impl, tuple.from);
- assertSame("Same instance for to", v3Impl, tuple.to);
-
+ @Override
+ public DataMigrationManager getDataMigrationManager() {
+ return dataMigrationManager;
}
- private MvccEntity getEntity(final String type){
-
- final SimpleId entityId = new SimpleId( type );
- final UUID version = UUIDGenerator.newTimeUUID();
- final Entity entity = new Entity( entityId );
-
- MvccEntityImpl logEntry = new MvccEntityImpl( entityId, version, MvccEntity.Status.COMPLETE, entity );
-
-
- return logEntry;
+ @Override
+ protected MvccEntitySerializationStrategy getExpectedSourceImpl() {
+ return v1Impl;
+ }
+ @Override
+ protected MvccEntitySerializationStrategy getExpectedTargetImpl() {
+ return v3Impl;
}
-
+ @Override
+ protected CollectionDataVersions getSourceVersion() {
+ return CollectionDataVersions.INITIAL;
+ }
@Override
- public DataMigrationManager getDataMigrationManager() {
- return dataMigrationManager;
+ protected CollectionDataVersions expectedTargetVersion() {
+ return CollectionDataVersions.LOG_REMOVAL;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/48127152/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/MvccEntityDataMigrationV2ToV3ImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/MvccEntityDataMigrationV2ToV3ImplTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/MvccEntityDataMigrationV2ToV3ImplTest.java
new file mode 100644
index 0000000..165105d
--- /dev/null
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/MvccEntityDataMigrationV2ToV3ImplTest.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence.collection.serialization.impl.migration;
+
+
+import org.junit.Rule;
+import org.junit.runner.RunWith;
+
+import org.apache.usergrid.persistence.collection.guice.TestCollectionModule;
+import org.apache.usergrid.persistence.collection.serialization.MvccEntitySerializationStrategy;
+import org.apache.usergrid.persistence.collection.serialization.impl.CollectionDataVersions;
+import org.apache.usergrid.persistence.collection.serialization.impl.MvccEntitySerializationStrategyV1Impl;
+import org.apache.usergrid.persistence.collection.serialization.impl.MvccEntitySerializationStrategyV2Impl;
+import org.apache.usergrid.persistence.collection.serialization.impl.MvccEntitySerializationStrategyV3Impl;
+import org.apache.usergrid.persistence.core.guice.MigrationManagerRule;
+import org.apache.usergrid.persistence.core.migration.data.DataMigrationManager;
+import org.apache.usergrid.persistence.core.migration.data.newimpls.VersionedMigrationSet;
+import org.apache.usergrid.persistence.core.test.ITRunner;
+import org.apache.usergrid.persistence.core.test.UseModules;
+
+import com.google.inject.Inject;
+
+import net.jcip.annotations.NotThreadSafe;
+
+
+@NotThreadSafe
+@RunWith( ITRunner.class )
+@UseModules( { TestCollectionModule.class } )
+public class MvccEntityDataMigrationV2ToV3ImplTest extends AbstractMvccEntityDataMigrationV1ToV3ImplTest{
+
+
+ @Inject
+ @Rule
+ public MigrationManagerRule migrationManagerRule;
+
+
+ @Inject
+ public DataMigrationManager dataMigrationManager;
+
+ @Inject
+ private MvccEntitySerializationStrategyV2Impl v2Impl;
+
+ @Inject
+ private MvccEntitySerializationStrategyV3Impl v3Impl;
+
+ @Inject
+ public MvccEntityDataMigrationImpl mvccEntityDataMigrationImpl;
+
+
+ @Inject
+ public VersionedMigrationSet<MvccEntitySerializationStrategy> versions;
+
+
+
+ @Override
+ public DataMigrationManager getDataMigrationManager() {
+ return dataMigrationManager;
+ }
+
+
+ @Override
+ protected MvccEntitySerializationStrategy getExpectedSourceImpl() {
+ return v2Impl;
+ }
+
+
+ @Override
+ protected MvccEntitySerializationStrategy getExpectedTargetImpl() {
+ return v3Impl;
+ }
+
+
+ @Override
+ protected CollectionDataVersions getSourceVersion() {
+ return CollectionDataVersions.BUFFER_SHORT_FIX;
+ }
+
+
+ @Override
+ protected CollectionDataVersions expectedTargetVersion() {
+ return CollectionDataVersions.LOG_REMOVAL;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/48127152/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/guice/MigrationManagerRule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/guice/MigrationManagerRule.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/guice/MigrationManagerRule.java
index cba88aa..ac6c169 100644
--- a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/guice/MigrationManagerRule.java
+++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/guice/MigrationManagerRule.java
@@ -27,6 +27,13 @@ public class MigrationManagerRule extends ExternalResource {
@Inject
public void setMigrationManager( final MigrationManager migrationManager ) {
this.migrationManager = migrationManager;
+
+ try {
+ this.migrationManager.migrate();
+ }
+ catch ( MigrationException e ) {
+ throw new RuntimeException(e);
+ }
}
@Inject
[33/50] [abbrv] incubator-usergrid git commit: Added REST tier
invocation for migrations.
Posted by sf...@apache.org.
Added REST tier invocation for migrations.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/1eca2718
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/1eca2718
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/1eca2718
Branch: refs/heads/USERGRID-480
Commit: 1eca2718df182dc93bbb269f20f3abfd467756a7
Parents: aa01d73
Author: Todd Nine <tn...@apigee.com>
Authored: Mon Mar 9 14:56:10 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Mon Mar 9 14:56:10 2015 -0600
----------------------------------------------------------------------
.../corepersistence/CpEntityManagerFactory.java | 1 -
.../apache/usergrid/rest/MigrateResource.java | 52 +++++++++++++++-----
2 files changed, 39 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1eca2718/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index 45d94d4..c58cd5e 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -131,7 +131,6 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
this.counterUtils = counterUtils;
this.injector = injector;
this.managerCache = injector.getInstance( ManagerCache.class );
- this.dataMigrationManager = injector.getInstance( DataMigrationManager.class );
this.metricsFactory = injector.getInstance( MetricsFactory.class );
this.orgApplicationCache = new OrgApplicationCacheImpl( this );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1eca2718/stack/rest/src/main/java/org/apache/usergrid/rest/MigrateResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/MigrateResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/MigrateResource.java
index 702d134..a6d2daa 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/MigrateResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/MigrateResource.java
@@ -20,6 +20,7 @@ package org.apache.usergrid.rest;
import java.util.Map;
+import java.util.Set;
import javax.ws.rs.DefaultValue;
import javax.ws.rs.GET;
@@ -37,6 +38,7 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component;
+import org.apache.usergrid.persistence.core.migration.data.DataMigrationManager;
import org.apache.usergrid.persistence.core.migration.schema.MigrationManager;
import org.apache.usergrid.rest.security.annotations.RequireSystemAccess;
@@ -62,6 +64,7 @@ public class MigrateResource extends AbstractContextResource {
logger.info( "SystemResource initialized" );
}
+
@Autowired
private Injector guiceInjector;
@@ -78,8 +81,6 @@ public class MigrateResource extends AbstractContextResource {
//TODO make this use the task scheduler
-
-
final Thread migrate = new Thread() {
@Override
@@ -106,7 +107,6 @@ public class MigrateResource extends AbstractContextResource {
}
-
@RequireSystemAccess
@PUT
@Path( "set" )
@@ -118,17 +118,23 @@ public class MigrateResource extends AbstractContextResource {
Preconditions.checkNotNull( json, "You must provide a json body" );
- Preconditions.checke
- String version = ( String ) json.get( "version" );
+ Preconditions.checkArgument( json.keySet().size() > 0, "You must specify at least one module and version" );
- Preconditions
- .checkArgument( version != null && version.length() > 0, "You must specify a version field in your json" );
+ /**
+ * Set the migration version for the plugins specified
+ */
+ for ( final String key : json.keySet() ) {
+ String version = ( String ) json.get( key );
+ Preconditions.checkArgument( version != null && version.length() > 0,
+ "You must specify a version field per module name" );
- int intVersion = Integer.parseInt( version );
- emf.setMigrationVersion( intVersion );
+ int intVersion = Integer.parseInt( version );
+
+ getDataMigrationManager().resetToVersion( key, intVersion );
+ }
return migrateStatus( ui, callback );
@@ -146,9 +152,16 @@ public class MigrateResource extends AbstractContextResource {
response.setAction( "Migrate Schema indexes" );
ObjectNode node = JsonNodeFactory.instance.objectNode();
- node.put( "currentVersion", emf.getMigrateDataVersion() );
- node.put( "lastMessage", emf.getMigrateDataStatus() );
- response.setProperty( "status", node );
+
+
+
+ final DataMigrationManager dataMigrationManager = getDataMigrationManager();
+
+ final Set<String> plugins = dataMigrationManager.getPluginNames();
+
+ for(final String pluginName: plugins){
+ node.put( pluginName, dataMigrationManager.getCurrentVersion( pluginName ) );
+ }
response.setSuccess();
@@ -173,7 +186,20 @@ public class MigrateResource extends AbstractContextResource {
return new JSONWithPadding( response, callback );
}
- private MigrationManager getMigrationManager(){
+
+ /**
+ * Get the Data migraiton manager
+ */
+ private DataMigrationManager getDataMigrationManager() {
+ return guiceInjector.getInstance( DataMigrationManager.class );
+ }
+
+
+ /**
+ * Get the Data migraiton manager
+ */
+ private MigrationManager getMigrationManager() {
return guiceInjector.getInstance( MigrationManager.class );
}
}
+
[35/50] [abbrv] incubator-usergrid git commit: merging from 472
Posted by sf...@apache.org.
merging from 472
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/5c7a5f86
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/5c7a5f86
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/5c7a5f86
Branch: refs/heads/USERGRID-480
Commit: 5c7a5f86aebd159782717e4ef29b3fada8cc1df0
Parents: 1eca271 e5cab76
Author: Shawn Feldman <sf...@apache.org>
Authored: Tue Mar 10 15:11:24 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Tue Mar 10 15:11:24 2015 -0600
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 111 +++++++-----
.../usergrid/persistence/map/MapManager.java | 8 +
.../persistence/map/impl/MapManagerImpl.java | 6 +
.../persistence/map/impl/MapSerialization.java | 20 ++-
.../map/impl/MapSerializationImpl.java | 85 +++++++--
.../persistence/map/MapManagerTest.java | 49 +++++-
stack/corepersistence/queryindex/pom.xml | 16 +-
.../persistence/index/guice/IndexModule.java | 4 +
.../index/impl/EsEntityIndexBatchImpl.java | 14 +-
.../index/impl/EsEntityIndexFactoryImpl.java | 12 +-
.../index/impl/EsEntityIndexImpl.java | 92 +++++++---
.../index/impl/EsIndexBufferConsumerImpl.java | 15 +-
.../persistence/index/impl/IndexingUtils.java | 111 ++++++------
.../persistence/index/utils/StringUtils.java | 126 +-------------
.../persistence/index/utils/UUIDUtils.java | 4 -
.../persistence/index/impl/EntityIndexTest.java | 173 ++++++++++++++++++-
16 files changed, 552 insertions(+), 294 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5c7a5f86/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/MapManager.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5c7a5f86/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
[34/50] [abbrv] incubator-usergrid git commit: fix for list types
changing from string to int or whatever
Posted by sf...@apache.org.
fix for list types changing from string to int or whatever
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/e5cab76f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/e5cab76f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/e5cab76f
Branch: refs/heads/USERGRID-480
Commit: e5cab76f8b8fc1e4a60ea23274e7a8a0cc79f759
Parents: ce14542
Author: Shawn Feldman <sf...@apache.org>
Authored: Tue Mar 10 15:01:54 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Tue Mar 10 15:01:54 2015 -0600
----------------------------------------------------------------------
.../index/impl/EsEntityIndexBatchImpl.java | 14 +--
.../persistence/index/impl/IndexingUtils.java | 109 ++++++++++---------
.../persistence/index/impl/EntityIndexTest.java | 38 ++++++-
3 files changed, 91 insertions(+), 70 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e5cab76f/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
index d987b29..18cc4fc 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
@@ -56,15 +56,7 @@ import com.codahale.metrics.Timer;
import rx.Observable;
import rx.functions.Func1;
-import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ANALYZED_STRING_PREFIX;
-import static org.apache.usergrid.persistence.index.impl.IndexingUtils.BOOLEAN_PREFIX;
-import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ENTITYID_ID_FIELDNAME;
-import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ENTITY_CONTEXT_FIELDNAME;
-import static org.apache.usergrid.persistence.index.impl.IndexingUtils.GEO_PREFIX;
-import static org.apache.usergrid.persistence.index.impl.IndexingUtils.NUMBER_PREFIX;
-import static org.apache.usergrid.persistence.index.impl.IndexingUtils.STRING_PREFIX;
-import static org.apache.usergrid.persistence.index.impl.IndexingUtils.createContextName;
-import static org.apache.usergrid.persistence.index.impl.IndexingUtils.createIndexDocId;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.*;
public class EsEntityIndexBatchImpl implements EntityIndexBatch {
@@ -255,7 +247,7 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
if ( f instanceof ListField ) {
List list = ( List ) field.getValue();
- entityMap.put( field.getName().toLowerCase(),
+ entityMap.put(LIST_PREFIX + field.getName().toLowerCase(),
new ArrayList( processCollectionForMap( list ) ) );
if ( !list.isEmpty() ) {
@@ -267,7 +259,7 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
}
else if ( f instanceof ArrayField ) {
List list = ( List ) field.getValue();
- entityMap.put( field.getName().toLowerCase(),
+ entityMap.put(ARRAY_PREFIX + field.getName().toLowerCase(),
new ArrayList( processCollectionForMap( list ) ) );
}
else if ( f instanceof SetField ) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e5cab76f/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
index ffd98e9..41a4680 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
@@ -34,6 +34,9 @@ public class IndexingUtils {
public static final String STRING_PREFIX = "su_";
public static final String ANALYZED_STRING_PREFIX = "sa_";
+ public static final String ARRAY_PREFIX = "ar_";
+ public static final String LIST_PREFIX = "li_";
+
public static final String GEO_PREFIX = "go_";
public static final String NUMBER_PREFIX = "nu_";
public static final String BOOLEAN_PREFIX = "bu_";
@@ -153,75 +156,75 @@ public class IndexingUtils {
/** add routing "_routing":{ "required":false, "path":"ug_entityId" **/
.startObject("_routing").field("required",true).field("path",ENTITYID_ID_FIELDNAME).endObject()
- .startArray("dynamic_templates")
+ .startArray("dynamic_templates")
// we need most specific mappings first since it's a stop on match algorithm
- .startObject()
+ .startObject()
- .startObject( "entity_id_template" )
- .field( "match", IndexingUtils.ENTITYID_ID_FIELDNAME )
- .field( "match_mapping_type", "string" )
- .startObject( "mapping" ).field( "type", "string" )
- .field( "index", "not_analyzed" )
- .endObject()
- .endObject()
- .endObject()
+ .startObject("entity_id_template" )
+ .field("match", IndexingUtils.ENTITYID_ID_FIELDNAME )
+ .field("match_mapping_type", "string" )
+ .startObject( "mapping").field("type", "string" )
+ .field("index", "not_analyzed" )
+ .endObject()
+ .endObject()
+ .endObject()
- .startObject()
- .startObject( "entity_context_template" )
+ .startObject()
+ .startObject("entity_context_template" )
.field( "match", IndexingUtils.ENTITY_CONTEXT_FIELDNAME )
- .field( "match_mapping_type", "string" )
- .startObject( "mapping" )
+ .field("match_mapping_type", "string" )
+ .startObject( "mapping" )
.field( "type", "string" )
- .field( "index", "not_analyzed" ).endObject()
- .endObject()
- .endObject()
-
- .startObject()
- .startObject( "entity_version_template" )
- .field( "match", IndexingUtils.ENTITY_VERSION_FIELDNAME )
- .field( "match_mapping_type", "string" )
- .startObject( "mapping" ).field( "type", "long" )
- .endObject()
- .endObject()
- .endObject()
+ .field("index", "not_analyzed").endObject()
+ .endObject()
+ .endObject()
+
+ .startObject()
+ .startObject("entity_version_template" )
+ .field("match", IndexingUtils.ENTITY_VERSION_FIELDNAME )
+ .field("match_mapping_type", "string" )
+ .startObject( "mapping").field("type", "long" )
+ .endObject()
+ .endObject()
+ .endObject()
// any string with field name that starts with sa_ gets analyzed
- .startObject()
- .startObject( "template_1" )
+ .startObject()
+ .startObject("template_1" )
.field( "match", ANALYZED_STRING_PREFIX + "*" )
- .field( "match_mapping_type", "string" ).startObject( "mapping" )
+ .field("match_mapping_type", "string" ).startObject( "mapping" )
.field( "type", "string" )
- .field( "index", "analyzed" )
- .endObject()
- .endObject()
+ .field("index", "analyzed" )
+ .endObject()
+ .endObject()
- .endObject()
+ .endObject()
// all other strings are not analyzed
- .startObject()
- .startObject( "template_2" )
+ .startObject()
+ .startObject("template_2" )
//todo, should be string prefix, remove 2 field mapping
- .field( "match", "*" )
- .field( "match_mapping_type", "string" )
- .startObject( "mapping" )
- .field( "type", "string" )
- .field( "index", "not_analyzed" )
- .endObject()
- .endObject()
- .endObject()
+ .field("match", "*" )
+ .field("match_mapping_type", "string" )
+ .startObject( "mapping" )
+ .field("type", "string" )
+ .field("index", "not_analyzed" )
+ .endObject()
+ .endObject()
+ .endObject()
// fields names starting with go_ get geo-indexed
- .startObject()
- .startObject( "template_3" )
- .field( "match", GEO_PREFIX + "location" )
- .startObject( "mapping" )
- .field( "type", "geo_point" )
- .endObject()
- .endObject()
- .endObject()
-
- .endArray()
+ .startObject()
+ .startObject("template_3" )
+ .field("match", GEO_PREFIX + "location" )
+ .startObject( "mapping" )
+ .field("type", "geo_point" )
+ .endObject()
+ .endObject()
+ .endObject()
+
+ .endArray()
.endObject();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e5cab76f/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
index a15053c..e4043d2 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
@@ -25,9 +25,7 @@ import java.util.concurrent.CountDownLatch;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.usergrid.persistence.index.*;
-import org.apache.usergrid.persistence.model.field.ArrayField;
-import org.apache.usergrid.persistence.model.field.EntityObjectField;
-import org.apache.usergrid.persistence.model.field.UUIDField;
+import org.apache.usergrid.persistence.model.field.*;
import org.apache.usergrid.persistence.model.field.value.EntityObject;
import org.junit.Ignore;
import org.junit.Test;
@@ -49,7 +47,6 @@ import org.apache.usergrid.persistence.index.utils.UUIDUtils;
import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.entity.SimpleId;
-import org.apache.usergrid.persistence.model.field.StringField;
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
import com.fasterxml.jackson.core.type.TypeReference;
@@ -90,7 +87,7 @@ public class EntityIndexTest extends BaseIT {
}
@Test
- @Ignore("this is a problem i will work on when i can breathe")
+// @Ignore("this is a problem i will work on when i can breathe")
public void testIndexVariations() throws IOException {
Id appId = new SimpleId( "application" );
@@ -110,6 +107,8 @@ public class EntityIndexTest extends BaseIT {
batch.index(indexScope, entity);
batch.execute().get();
+ entity = new Entity( entityType );
+ entity.setField(new UUIDField(IndexingUtils.ENTITYID_ID_FIELDNAME, UUID.randomUUID()));
EntityUtils.setVersion(entity, UUIDGenerator.newTimeUUID());
List<String> list = new ArrayList<>();
list.add("test");
@@ -117,6 +116,9 @@ public class EntityIndexTest extends BaseIT {
batch.index(indexScope, entity);
batch.execute().get();
+
+ entity = new Entity( entityType );
+ entity.setField(new UUIDField(IndexingUtils.ENTITYID_ID_FIELDNAME, UUID.randomUUID()));
EntityUtils.setVersion(entity, UUIDGenerator.newTimeUUID());
EntityObject testObj = new EntityObject();
testObj.setField(new StringField("test","testFiedl"));
@@ -124,9 +126,32 @@ public class EntityIndexTest extends BaseIT {
batch.index(indexScope, entity);
batch.execute().get();
+ entity = new Entity( entityType );
+ entity.setField(new UUIDField(IndexingUtils.ENTITYID_ID_FIELDNAME, UUID.randomUUID()));
+ EntityUtils.setVersion(entity, UUIDGenerator.newTimeUUID());
+ List<Integer> listint = new ArrayList<>();
+ listint.add(0);
+ entity.setField(new ArrayField<Integer>("testfield", listint));
+ batch.index(indexScope, entity);
+ batch.execute().get();
+
+ entity = new Entity( entityType );
+ entity.setField(new UUIDField(IndexingUtils.ENTITYID_ID_FIELDNAME, UUID.randomUUID()));
+ EntityUtils.setVersion(entity, UUIDGenerator.newTimeUUID());
+ List<EntityObject> listObj = new ArrayList<>();
+ EntityObject listObjField = new EntityObject();
+ listObjField.setField(new StringField("testasf","somevalue"));
+ listObj.add(listObjField);
+ listObjField = new EntityObject();
+ listObjField.setField(new IntegerField("testasf",0));
+ listObj.add(listObjField);
+ entity.setField(new ArrayField<EntityObject>("testfield", listObj));
+ batch.index(indexScope, entity);
+ batch.execute().get();
+
entityIndex.refresh();
+ testQuery(indexScope, searchTypes, entityIndex, "select *", 5);
- testQueries( indexScope, searchTypes, entityIndex );
}
@Test
@@ -254,6 +279,7 @@ public class EntityIndexTest extends BaseIT {
entityIndex.refresh();
//Hilda Youn
+
testQuery(indexScope, searchTypes, entityIndex, "name = 'Bowers Oneil'", 0);
}
[31/50] [abbrv] incubator-usergrid git commit: merge from 416
Posted by sf...@apache.org.
merge from 416
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/b6351134
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/b6351134
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/b6351134
Branch: refs/heads/USERGRID-480
Commit: b6351134e6d9eb0e1def02fe319aeb3b81db29ad
Parents: 0c3b439 673c057
Author: Shawn Feldman <sf...@apache.org>
Authored: Mon Mar 9 09:25:03 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Mon Mar 9 09:25:03 2015 -0600
----------------------------------------------------------------------
.../Client/IUsergridClient.cs | 2 +-
.../Usergrid.Notifications/Client/PushClient.cs | 24 +++
.../Usergrid.Notifications/MainPage.xaml | 1 +
.../Usergrid.Notifications/MainPage.xaml.cs | 29 ++-
.../main/dist/init_instance/init_rest_server.sh | 3 -
.../src/main/groovy/configure_usergrid.groovy | 6 +-
stack/awscluster/ugcluster-cf.json | 8 +
.../corepersistence/CpEntityManager.java | 109 +++++++++-
.../corepersistence/CpEntityManagerFactory.java | 151 ++++++--------
.../corepersistence/CpRelationManager.java | 39 +++-
.../corepersistence/OrgApplicationCache.java | 67 +++++++
.../OrgApplicationCacheImpl.java | 181 +++++++++++++++++
.../corepersistence/results/EntityVerifier.java | 4 +-
.../results/FilteringLoader.java | 2 +-
.../cassandra/CassandraPersistenceUtils.java | 2 +-
.../main/resources/usergrid-core-context.xml | 14 +-
.../cassandra/EntityManagerFactoryImplIT.java | 1 +
.../impl/migration/GraphMigrationPlugin.java | 1 -
.../persistence/index/IndexBufferProducer.java | 10 +-
.../usergrid/persistence/index/IndexFig.java | 12 +-
.../index/impl/EsEntityIndexBatchImpl.java | 15 +-
.../index/impl/EsEntityIndexFactoryImpl.java | 9 +-
.../index/impl/EsEntityIndexImpl.java | 199 +++++++++++++++----
.../index/impl/EsIndexBufferConsumerImpl.java | 39 +++-
.../index/impl/EsIndexBufferProducerImpl.java | 30 ++-
.../persistence/index/impl/IndexingUtils.java | 7 +-
.../persistence/index/impl/EntityIndexTest.java | 2 +-
.../datagenerators/EntityDataGenerator.scala | 25 +++
.../datagenerators/FeederGenerator.scala | 18 +-
.../usergrid/scenarios/EntityScenarios.scala | 8 +-
.../org/apache/usergrid/settings/Settings.scala | 4 +-
.../simulations/PutCustomEntitySimulation.scala | 75 +++++++
.../org/apache/usergrid/rest/IndexResource.java | 19 ++
.../services/notifications/wns/WNSAdapter.java | 39 ++++
.../resources/usergrid-services-context.xml | 8 +-
35 files changed, 947 insertions(+), 216 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b6351134/stack/awscluster/src/main/dist/init_instance/init_rest_server.sh
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b6351134/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index 3b39bc9,12504ed..a897ad2
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@@ -41,9 -46,8 +49,10 @@@ import org.apache.usergrid.persistence.
import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.collection.EntityCollectionManager;
import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl;
+ import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
+import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
import org.apache.usergrid.persistence.core.migration.data.DataMigrationManager;
+import org.apache.usergrid.persistence.core.migration.data.MigrationDataProvider;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
import org.apache.usergrid.persistence.core.util.Health;
@@@ -106,9 -97,10 +111,12 @@@ public class CpEntityManagerFactory imp
}
});
+ private final OrgApplicationCache orgApplicationCache;
+
+
private ManagerCache managerCache;
+
+
private DataMigrationManager dataMigrationManager;
private CassandraService cassandraService;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b6351134/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/GraphMigrationPlugin.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/GraphMigrationPlugin.java
index 22f34ff,0000000..ed51c27
mode 100644,000000..100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/GraphMigrationPlugin.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/GraphMigrationPlugin.java
@@@ -1,70 -1,0 +1,69 @@@
+/*
+ *
+ * *
+ * * Licensed to the Apache Software Foundation (ASF) under one
+ * * or more contributor license agreements. See the NOTICE file
+ * * distributed with this work for additional information
+ * * regarding copyright ownership. The ASF licenses this file
+ * * to you under the Apache License, Version 2.0 (the
+ * * "License"); you may not use this file except in compliance
+ * * with the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing,
+ * * software distributed under the License is distributed on an
+ * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * * KIND, either express or implied. See the License for the
+ * * specific language governing permissions and limitations
+ * * under the License.
+ * *
+ *
+ */
+
+package org.apache.usergrid.persistence.graph.serialization.impl.migration;
+
+
+import java.util.Set;
+
+import org.apache.usergrid.persistence.core.migration.data.MigrationInfoSerialization;
+import org.apache.usergrid.persistence.core.migration.data.AbstractMigrationPlugin;
+import org.apache.usergrid.persistence.core.migration.data.DataMigration;
+import org.apache.usergrid.persistence.core.migration.data.MigrationDataProvider;
+import org.apache.usergrid.persistence.core.migration.data.PluginPhase;
+
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+
- import sun.plugin2.main.server.Plugin;
+
+
+/**
+ * Migration plugin for the collection module
+ */
+@Singleton
+public class GraphMigrationPlugin extends AbstractMigrationPlugin<GraphNode> {
+
+ public static final String PLUGIN_NAME = "graph-data";
+
+
+
+ @Inject
+ public GraphMigrationPlugin(@GraphMigration
+ final Set<DataMigration<GraphNode>> entityDataMigrations,
+ final MigrationDataProvider<GraphNode> entityIdScopeDataMigrationProvider,
+ final MigrationInfoSerialization migrationInfoSerialization ) {
+ super( entityDataMigrations, entityIdScopeDataMigrationProvider, migrationInfoSerialization );
+ }
+
+
+ @Override
+ public String getName() {
+ return PLUGIN_NAME;
+ }
+
+
+ @Override
+ public PluginPhase getPhase() {
+ return PluginPhase.MIGRATE;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b6351134/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index b1794f8,c9f5590..caa2fc7
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@@ -44,11 -48,9 +48,12 @@@ import org.elasticsearch.action.admin.c
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksRequest;
import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksResponse;
+import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse;
+import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
+import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse;
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse;
+ import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse;
import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateResponse;
import org.elasticsearch.action.deletebyquery.DeleteByQueryResponse;
import org.elasticsearch.action.deletebyquery.IndexDeleteByQueryResponse;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b6351134/stack/rest/src/main/java/org/apache/usergrid/rest/IndexResource.java
----------------------------------------------------------------------
[28/50] [abbrv] incubator-usergrid git commit: Fixes bug in return
type
Posted by sf...@apache.org.
Fixes bug in return type
Adds test for migration
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/74a6c35a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/74a6c35a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/74a6c35a
Branch: refs/heads/USERGRID-480
Commit: 74a6c35acfefb8f7615c36cb2887377547a7aab3
Parents: dc87edc
Author: Todd Nine <tn...@apigee.com>
Authored: Sat Mar 7 10:31:41 2015 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Sat Mar 7 10:31:41 2015 -0700
----------------------------------------------------------------------
.../migration/EntityTypeMappingMigration.java | 7 +-
.../migration/EntityTypeMappingMigrationIT.java | 204 ++++++++-----------
2 files changed, 92 insertions(+), 119 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/74a6c35a/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/EntityTypeMappingMigration.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/EntityTypeMappingMigration.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/EntityTypeMappingMigration.java
index ad50c18..40ad236 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/EntityTypeMappingMigration.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/EntityTypeMappingMigration.java
@@ -63,7 +63,7 @@ public class EntityTypeMappingMigration implements DataMigration<EntityIdScope>
final AtomicLong atomicLong = new AtomicLong();
- return allEntitiesInSystemObservable.getData()
+ allEntitiesInSystemObservable.getData()
//process the entities in parallel
.parallel( new Func1<Observable<EntityIdScope>, Observable<EntityIdScope>>() {
@@ -93,6 +93,11 @@ public class EntityTypeMappingMigration implements DataMigration<EntityIdScope>
} );
}
} ).count().toBlocking().last();
+
+
+ return getMaxVersion();
+
+
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/74a6c35a/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/EntityTypeMappingMigrationIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/EntityTypeMappingMigrationIT.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/EntityTypeMappingMigrationIT.java
index 35608d5..be7cee4 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/EntityTypeMappingMigrationIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/EntityTypeMappingMigrationIT.java
@@ -20,140 +20,108 @@
package org.apache.usergrid.corepersistence.migration;
-import org.junit.Before;
-import org.junit.Ignore;
import org.junit.Test;
import org.apache.usergrid.AbstractCoreIT;
import org.apache.usergrid.corepersistence.ManagerCache;
-import org.apache.usergrid.persistence.EntityManagerFactory;
-import org.apache.usergrid.persistence.core.migration.data.DataMigrationManager;
+import org.apache.usergrid.corepersistence.util.CpNamingUtils;
+import org.apache.usergrid.persistence.collection.CollectionScope;
+import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl;
+import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
+import org.apache.usergrid.persistence.core.migration.data.TestMigrationDataProvider;
+import org.apache.usergrid.persistence.core.migration.data.TestProgressObserver;
+import org.apache.usergrid.persistence.map.MapManager;
+import org.apache.usergrid.persistence.map.MapScope;
+import org.apache.usergrid.persistence.map.impl.MapScopeImpl;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import rx.Observable;
+
+import static org.apache.usergrid.persistence.core.util.IdGenerator.createId;
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+
+/**
+ * Test for our entity type mapping
+ */
+public class EntityTypeMappingMigrationIT {
-import com.google.inject.Injector;
-import com.netflix.astyanax.Keyspace;
-import net.jcip.annotations.NotThreadSafe;
+ @Test
+ public void testIdMapping() throws Throwable {
+ final Id applicationId = createId("application");
-@NotThreadSafe
-public class EntityTypeMappingMigrationIT extends AbstractCoreIT {
+ final CollectionScope scope1 = new CollectionScopeImpl( applicationId, applicationId, "things" );
+ final Id entityId1 = createId("thing");
- private Injector injector;
+ final EntityIdScope idScope1 = new EntityIdScope(scope1, entityId1 );
+ final MapScope mapScope1 = new MapScopeImpl(applicationId, CpNamingUtils.TYPES_BY_UUID_MAP );
- private EntityTypeMappingMigration entityTypeMappingMigration;
- private Keyspace keyspace;
- private EntityManagerFactory emf;
- private ManagerCache managerCache;
- private DataMigrationManager dataMigrationManager;
-// /**
-// * Rule to do the resets we need
-// */
-// @Rule
-// public MigrationTestRule migrationTestRule = new MigrationTestRule(
-// app, SpringResource.getInstance().getBean( Injector.class ) ,EntityTypeMappingMigration.class );
-// private AllEntitiesInSystemObservable allEntitiesInSystemObservable;
+ final Id subCollectionOwner = createId("owners");
+ final CollectionScope scope2 = new CollectionScopeImpl( applicationId, subCollectionOwner, "foos" );
- //TODO USERGRID-405
- @Before
- public void setup() {
-// injector = SpringResource.getInstance().getBean( Injector.class );
-// emf = setup.getEmf();
-// entityTypeMappingMigration = injector.getInstance( EntityTypeMappingMigration.class );
-// keyspace = injector.getInstance( Keyspace.class );
-// managerCache = injector.getInstance( ManagerCache.class );
-// dataMigrationManager = injector.getInstance( DataMigrationManager.class );
-// allEntitiesInSystemObservable = injector.getInstance(AllEntitiesInSystemObservable.class);
- }
+ final Id entityId2 = createId("foo");
+ final EntityIdScope idScope2 = new EntityIdScope( scope2, entityId2 );
+
+ final MapScope mapScope2 = new MapScopeImpl(applicationId, CpNamingUtils.TYPES_BY_UUID_MAP );
+
+
+ final Observable<EntityIdScope> scopes = Observable.from(idScope1, idScope2);
+
+ final TestMigrationDataProvider<EntityIdScope> migrationDataProvider = new TestMigrationDataProvider<>();
+
+ //set our scopes
+ migrationDataProvider.setObservable( scopes );
+
+
+
+
+ //mock up returning our map manager
+ final MapManager mapManager = mock(MapManager.class);
+ final ManagerCache managerCache = mock(ManagerCache.class);
+
+ when(managerCache.getMapManager( eq( mapScope1 ) )).thenReturn( mapManager );
+
+ when(managerCache.getMapManager( eq( mapScope2 ) )).thenReturn( mapManager );
+
+ final TestProgressObserver progressObserver = new TestProgressObserver();
- @Test
- @Ignore("Ignored awaiting fix for USERGRID-268")
- public void testIdMapping() throws Throwable {
-// assertEquals( "version 1 expected", 1, entityTypeMappingMigration.getVersion() );
-// assertEquals( "Previous version expected", 0, dataMigrationManager.getCurrentVersion());
-//
-// final EntityManager newAppEm = app.getEntityManager();
-//
-// final String type1 = "type1thing";
-// final String type2 = "type2thing";
-// final int size = 10;
-//
-// final Set<Id> type1Identities = EntityWriteHelper.createTypes( newAppEm, type1, size );
-// final Set<Id> type2Identities = EntityWriteHelper.createTypes( newAppEm, type2, size );
-//
-//
-// final Set<Id> allEntities = new HashSet<>();
-// allEntities.addAll( type1Identities );
-// allEntities.addAll( type2Identities );
-//
-//
-// /**
-// * Drop our map keyspace to ensure we have no entries before migrating after doing our writes.
-// * This will ensure we have the data
-// */
-// keyspace.truncateColumnFamily( MapSerializationImpl.MAP_ENTRIES );
-// keyspace.truncateColumnFamily( MapSerializationImpl.MAP_KEYS );
-//
-// app.createApplication(
-// GraphShardVersionMigrationIT.class.getSimpleName()+ UUIDGenerator.newTimeUUID(),
-// "migrationTest" );
-//
-//
-//
-// final TestProgressObserver progressObserver = new TestProgressObserver();
-//
-// entityTypeMappingMigration.migrate(allEntitiesInSystemObservable.getAllEntitiesInSystem(1000), progressObserver).toBlocking().last();
-//
-// allEntitiesInSystemObservable.getAllEntitiesInSystem(1000)
-// .doOnNext(new Action1<ApplicationEntityGroup<CollectionScope>>() {
-// @Override
-// public void call(
-// final ApplicationEntityGroup<CollectionScope> entity) {
-// //ensure that each one has a type
-//
-// final EntityManager em = emf.getEntityManager(
-// entity.applicationScope.getApplication().getUuid());
-//
-// for (final EntityIdScope<CollectionScope> idScope : entity.entityIds) {
-// final Id id = idScope.getId();
-// try {
-// final Entity returned = em.get(id.getUuid());
-//
-// //we seem to occasionally get phantom edges. If this is the
-// // case we'll store the type _> uuid mapping, but we won't have
-// // anything to load
-//
-// if (returned != null) {
-// assertEquals(id.getUuid(), returned.getUuid());
-// assertEquals(id.getType(), returned.getType());
-// } else {
-// final String type = managerCache.getMapManager(CpNamingUtils
-// .getEntityTypeMapScope(
-// entity.applicationScope.getApplication()))
-// .getString(id.getUuid()
-// .toString());
-//
-// assertEquals(id.getType(), type);
-// }
-// } catch (Exception e) {
-// throw new RuntimeException("Unable to get entity " + id
-// + " by UUID, migration failed", e);
-// }
-//
-// allEntities.remove(id);
-// }
-// }
-// }).toBlocking().lastOrDefault(null);
-//
-//
-// assertEquals("Every element should have been encountered", 0, allEntities.size());
-// assertFalse("Progress observer should not have failed", progressObserver.getFailed());
-// assertTrue("Progress observer should have update messages", progressObserver.getUpdates().size() > 0);
- }
- }
+ //wire it up
+ final EntityTypeMappingMigration migration = new EntityTypeMappingMigration( managerCache, migrationDataProvider );
+
+ //run it
+
+ final int returnedVersion = migration.migrate(CoreDataVersions.INITIAL.getVersion(), migrationDataProvider, progressObserver );
+
+
+ assertEquals(CoreDataVersions.ID_MAP_FIX.getVersion(), returnedVersion);
+
+ //verify we saved it
+
+ verify(mapManager).putString(entityId1.getUuid().toString(), entityId1.getType() );
+
+ verify(mapManager).putString(entityId2.getUuid().toString(), entityId2.getType() );
+
+
+
+
+
+
+
+
+
+
+ }
+}
[24/50] [abbrv] incubator-usergrid git commit: Refactored packages
Posted by sf...@apache.org.
Refactored packages
Fixes system setup migration wiring
Added more tests
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/cf32557a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/cf32557a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/cf32557a
Branch: refs/heads/USERGRID-480
Commit: cf32557a101b7d654f2d2d2090b385cb74110220
Parents: 494f8ae
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Mar 4 23:13:32 2015 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Mar 4 23:13:32 2015 -0700
----------------------------------------------------------------------
.../usergrid/corepersistence/CoreModule.java | 14 +-
.../corepersistence/CpEntityManagerFactory.java | 4 +-
.../migration/CoreMigration.java | 53 +++
.../migration/CoreMigrationPlugin.java | 8 +-
.../migration/EntityTypeMappingMigration.java | 8 +-
.../migration/MigrationModuleVersion.java | 8 +-
.../rx/impl/AbstractGraphVisitorImpl.java | 5 +-
.../MvccEntitySerializationStrategy.java | 2 +-
...vccEntitySerializationStrategyProxyImpl.java | 4 +-
.../serialization/impl/SerializationModule.java | 15 +-
.../impl/migration/CollectionMigration.java | 53 +++
.../migration/CollectionMigrationPlugin.java | 9 +-
.../migration/MvccEntityDataMigrationImpl.java | 13 +-
.../collection/guice/TestCollectionModule.java | 6 +-
...ntitySerializationStrategyProxyV1_3Test.java | 1 -
...ctMvccEntityDataMigrationV1ToV3ImplTest.java | 10 +-
.../MvccEntityDataMigrationV1ToV3ImplTest.java | 20 +-
.../MvccEntityDataMigrationV2ToV3ImplTest.java | 3 +-
.../persistence/core/guice/CommonModule.java | 2 +-
.../core/hystrix/HystrixCassandra.java | 30 +-
.../migration/data/AbstractMigrationPlugin.java | 128 +++++++
.../core/migration/data/DataMigration.java | 58 +++
.../data/DataMigrationManagerImpl.java | 21 +-
.../migration/data/MigrationDataProvider.java | 43 +++
.../core/migration/data/MigrationPlugin.java | 51 +++
.../migration/data/MigrationRelationship.java | 100 +++++
.../core/migration/data/ProgressObserver.java | 63 ++++
.../core/migration/data/VersionedData.java | 38 ++
.../migration/data/VersionedMigrationSet.java | 153 ++++++++
.../data/newimpls/AbstractMigrationPlugin.java | 126 -------
.../migration/data/newimpls/DataMigration2.java | 58 ---
.../data/newimpls/MigrationDataProvider.java | 43 ---
.../data/newimpls/MigrationPlugin.java | 51 ---
.../data/newimpls/MigrationRelationship.java | 100 -----
.../data/newimpls/ProgressObserver.java | 50 ---
.../migration/data/newimpls/VersionedData.java | 38 --
.../data/newimpls/VersionedMigrationSet.java | 153 --------
.../data/DataMigrationManagerImplTest.java | 366 +++++++------------
.../data/MigrationInfoSerializationTest.java | 112 +++---
.../data/TestMigrationDataProvider.java | 61 ++++
.../migration/data/TestProgressObserver.java | 24 +-
.../data/VersionedMigrationSetTest.java | 198 ++++++++++
.../newimpls/TestMigrationDataProvider.java | 61 ----
.../newimpls/VersionedMigrationSetTest.java | 194 ----------
.../persistence/graph/guice/GraphModule.java | 14 +-
.../EdgeMetadataSerialization.java | 2 +-
.../EdgeMetadataSerializationProxyImpl.java | 4 +-
.../impl/migration/EdgeDataMigrationImpl.java | 12 +-
.../impl/migration/GraphMigration.java | 53 +++
.../impl/migration/GraphMigrationPlugin.java | 9 +-
.../graph/guice/TestGraphModule.java | 5 +-
.../migration/EdgeDataMigrationImplTest.java | 4 +-
.../apache/usergrid/rest/MigrateResource.java | 4 +-
53 files changed, 1370 insertions(+), 1295 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cf32557a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
index 9f01feb..62be6cf 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
@@ -21,8 +21,10 @@ import org.springframework.context.ApplicationContext;
import org.apache.usergrid.corepersistence.events.EntityDeletedHandler;
import org.apache.usergrid.corepersistence.events.EntityVersionCreatedHandler;
import org.apache.usergrid.corepersistence.events.EntityVersionDeletedHandler;
+import org.apache.usergrid.corepersistence.migration.CoreMigration;
import org.apache.usergrid.corepersistence.migration.CoreMigrationPlugin;
import org.apache.usergrid.corepersistence.migration.EntityTypeMappingMigration;
+import org.apache.usergrid.corepersistence.migration.MigrationModuleVersion;
import org.apache.usergrid.corepersistence.rx.impl.AllEntitiesInSystemImpl;
import org.apache.usergrid.corepersistence.rx.impl.AllNodesInGraphImpl;
import org.apache.usergrid.corepersistence.rx.impl.AllApplicationsObservableImpl;
@@ -34,9 +36,9 @@ import org.apache.usergrid.persistence.collection.guice.CollectionModule;
import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
import org.apache.usergrid.persistence.collection.serialization.impl.migration.MvccEntityDataMigrationImpl;
import org.apache.usergrid.persistence.core.guice.CommonModule;
-import org.apache.usergrid.persistence.core.migration.data.newimpls.DataMigration2;
-import org.apache.usergrid.persistence.core.migration.data.newimpls.MigrationDataProvider;
-import org.apache.usergrid.persistence.core.migration.data.newimpls.MigrationPlugin;
+import org.apache.usergrid.persistence.core.migration.data.DataMigration;
+import org.apache.usergrid.persistence.core.migration.data.MigrationDataProvider;
+import org.apache.usergrid.persistence.core.migration.data.MigrationPlugin;
import org.apache.usergrid.persistence.graph.guice.GraphModule;
import org.apache.usergrid.persistence.graph.serialization.impl.migration.GraphNode;
import org.apache.usergrid.persistence.index.guice.IndexModule;
@@ -121,12 +123,12 @@ public class CoreModule extends AbstractModule {
* Create our migrations for within our core plugin
*
*/
- Multibinder<DataMigration2<EntityIdScope>> dataMigrationMultibinder =
- Multibinder.newSetBinder( binder(), new TypeLiteral<DataMigration2<EntityIdScope>>() {} );
+ Multibinder<DataMigration<EntityIdScope>> dataMigrationMultibinder =
+ Multibinder.newSetBinder( binder(), new TypeLiteral<DataMigration<EntityIdScope>>() {}, CoreMigration.class );
- dataMigrationMultibinder.addBinding().to( MvccEntityDataMigrationImpl.class );
dataMigrationMultibinder.addBinding().to( EntityTypeMappingMigration.class );
+ dataMigrationMultibinder.addBinding().to( MigrationModuleVersion.class );
//wire up the collection migration plugin
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cf32557a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index e0d7daa..3b39bc9 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -43,7 +43,7 @@ import org.apache.usergrid.persistence.collection.EntityCollectionManager;
import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl;
import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
import org.apache.usergrid.persistence.core.migration.data.DataMigrationManager;
-import org.apache.usergrid.persistence.core.migration.data.newimpls.MigrationDataProvider;
+import org.apache.usergrid.persistence.core.migration.data.MigrationDataProvider;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
import org.apache.usergrid.persistence.core.util.Health;
@@ -367,7 +367,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
this.rebuildApplicationIndexes(applicationId, new ProgressObserver() {
@Override
public void onProgress(EntityRef entity) {
- logger.info("Restored entity {}:{}", entity.getType(), entity.getUuid());
+ logger.info( "Restored entity {}:{}", entity.getType(), entity.getUuid() );
}
});
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cf32557a/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/CoreMigration.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/CoreMigration.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/CoreMigration.java
new file mode 100644
index 0000000..4df32da
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/CoreMigration.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.migration;/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+import java.lang.annotation.Retention;
+import java.lang.annotation.Target;
+
+import com.google.inject.BindingAnnotation;
+
+import static java.lang.annotation.ElementType.FIELD;
+import static java.lang.annotation.ElementType.METHOD;
+import static java.lang.annotation.ElementType.PARAMETER;
+import static java.lang.annotation.RetentionPolicy.RUNTIME;
+
+
+@BindingAnnotation
+@Target({ FIELD, PARAMETER, METHOD }) @Retention(RUNTIME)
+public @interface CoreMigration {}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cf32557a/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/CoreMigrationPlugin.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/CoreMigrationPlugin.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/CoreMigrationPlugin.java
index b95900f..91c1570 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/CoreMigrationPlugin.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/CoreMigrationPlugin.java
@@ -24,9 +24,9 @@ import java.util.Set;
import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
import org.apache.usergrid.persistence.core.migration.data.MigrationInfoSerialization;
-import org.apache.usergrid.persistence.core.migration.data.newimpls.AbstractMigrationPlugin;
-import org.apache.usergrid.persistence.core.migration.data.newimpls.DataMigration2;
-import org.apache.usergrid.persistence.core.migration.data.newimpls.MigrationDataProvider;
+import org.apache.usergrid.persistence.core.migration.data.AbstractMigrationPlugin;
+import org.apache.usergrid.persistence.core.migration.data.DataMigration;
+import org.apache.usergrid.persistence.core.migration.data.MigrationDataProvider;
import com.google.inject.Inject;
import com.google.inject.Singleton;
@@ -43,7 +43,7 @@ public class CoreMigrationPlugin extends AbstractMigrationPlugin<EntityIdScope>
@Inject
- public CoreMigrationPlugin( final Set<DataMigration2<EntityIdScope>> entityDataMigrations,
+ public CoreMigrationPlugin( @CoreMigration final Set<DataMigration<EntityIdScope>> entityDataMigrations,
final MigrationDataProvider<EntityIdScope> entityIdScopeDataMigrationProvider,
final MigrationInfoSerialization migrationInfoSerialization ) {
super( entityDataMigrations, entityIdScopeDataMigrationProvider, migrationInfoSerialization );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cf32557a/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/EntityTypeMappingMigration.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/EntityTypeMappingMigration.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/EntityTypeMappingMigration.java
index 80d7ebe..1a5ef13 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/EntityTypeMappingMigration.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/EntityTypeMappingMigration.java
@@ -26,9 +26,9 @@ import java.util.concurrent.atomic.AtomicLong;
import org.apache.usergrid.corepersistence.ManagerCache;
import org.apache.usergrid.corepersistence.util.CpNamingUtils;
import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
-import org.apache.usergrid.persistence.core.migration.data.newimpls.DataMigration2;
-import org.apache.usergrid.persistence.core.migration.data.newimpls.MigrationDataProvider;
-import org.apache.usergrid.persistence.core.migration.data.newimpls.ProgressObserver;
+import org.apache.usergrid.persistence.core.migration.data.DataMigration;
+import org.apache.usergrid.persistence.core.migration.data.MigrationDataProvider;
+import org.apache.usergrid.persistence.core.migration.data.ProgressObserver;
import org.apache.usergrid.persistence.map.MapManager;
import org.apache.usergrid.persistence.map.MapScope;
@@ -42,7 +42,7 @@ import rx.functions.Func1;
/**
* Migration to ensure that our entity id is written into our map data
*/
-public class EntityTypeMappingMigration implements DataMigration2<EntityIdScope> {
+public class EntityTypeMappingMigration implements DataMigration<EntityIdScope> {
private final ManagerCache managerCache;
private final MigrationDataProvider<EntityIdScope> allEntitiesInSystemObservable;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cf32557a/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/MigrationModuleVersion.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/MigrationModuleVersion.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/MigrationModuleVersion.java
index 1c124ac..9ddc219 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/MigrationModuleVersion.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/MigrationModuleVersion.java
@@ -24,9 +24,9 @@ import org.apache.usergrid.persistence.collection.serialization.impl.MvccEntityS
import org.apache.usergrid.persistence.collection.serialization.impl.migration.CollectionMigrationPlugin;
import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
import org.apache.usergrid.persistence.core.migration.data.MigrationInfoSerialization;
-import org.apache.usergrid.persistence.core.migration.data.newimpls.DataMigration2;
-import org.apache.usergrid.persistence.core.migration.data.newimpls.MigrationDataProvider;
-import org.apache.usergrid.persistence.core.migration.data.newimpls.ProgressObserver;
+import org.apache.usergrid.persistence.core.migration.data.DataMigration;
+import org.apache.usergrid.persistence.core.migration.data.MigrationDataProvider;
+import org.apache.usergrid.persistence.core.migration.data.ProgressObserver;
import org.apache.usergrid.persistence.graph.serialization.impl.EdgeMetadataSerializationV2Impl;
import org.apache.usergrid.persistence.graph.serialization.impl.migration.GraphMigrationPlugin;
@@ -37,7 +37,7 @@ import com.google.inject.Inject;
* Migration to set our module versions now that we've refactor for sub modules Keeps the EntityIdScope because it won't
* subscribe to the data provider.
*/
-public class MigrationModuleVersion implements DataMigration2<EntityIdScope> {
+public class MigrationModuleVersion implements DataMigration<EntityIdScope> {
/**
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cf32557a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AbstractGraphVisitorImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AbstractGraphVisitorImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AbstractGraphVisitorImpl.java
index 07b7a3d..b8cb80b 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AbstractGraphVisitorImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AbstractGraphVisitorImpl.java
@@ -21,10 +21,7 @@ package org.apache.usergrid.corepersistence.rx.impl;
import org.apache.usergrid.corepersistence.AllApplicationsObservable;
-import org.apache.usergrid.corepersistence.util.CpNamingUtils;
-import org.apache.usergrid.persistence.collection.CollectionScope;
-import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
-import org.apache.usergrid.persistence.core.migration.data.newimpls.MigrationDataProvider;
+import org.apache.usergrid.persistence.core.migration.data.MigrationDataProvider;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.graph.GraphManager;
import org.apache.usergrid.persistence.graph.GraphManagerFactory;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cf32557a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/MvccEntitySerializationStrategy.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/MvccEntitySerializationStrategy.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/MvccEntitySerializationStrategy.java
index de3cab3..7028620 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/MvccEntitySerializationStrategy.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/MvccEntitySerializationStrategy.java
@@ -26,7 +26,7 @@ import java.util.UUID;
import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.collection.EntitySet;
import org.apache.usergrid.persistence.collection.MvccEntity;
-import org.apache.usergrid.persistence.core.migration.data.newimpls.VersionedData;
+import org.apache.usergrid.persistence.core.migration.data.VersionedData;
import org.apache.usergrid.persistence.core.migration.schema.Migration;
import org.apache.usergrid.persistence.model.entity.Id;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cf32557a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyImpl.java
index 19bfc9e..b815ef4 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyImpl.java
@@ -29,8 +29,8 @@ import org.apache.usergrid.persistence.collection.serialization.MvccEntitySerial
import org.apache.usergrid.persistence.collection.serialization.impl.migration.CollectionMigrationPlugin;
import org.apache.usergrid.persistence.core.astyanax.MultiTennantColumnFamilyDefinition;
import org.apache.usergrid.persistence.core.migration.data.MigrationInfoCache;
-import org.apache.usergrid.persistence.core.migration.data.newimpls.MigrationRelationship;
-import org.apache.usergrid.persistence.core.migration.data.newimpls.VersionedMigrationSet;
+import org.apache.usergrid.persistence.core.migration.data.MigrationRelationship;
+import org.apache.usergrid.persistence.core.migration.data.VersionedMigrationSet;
import org.apache.usergrid.persistence.model.entity.Id;
import com.google.common.base.Optional;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cf32557a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/SerializationModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/SerializationModule.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/SerializationModule.java
index 241a274..9d6d7fb 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/SerializationModule.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/SerializationModule.java
@@ -21,20 +21,20 @@ package org.apache.usergrid.persistence.collection.serialization.impl;
import org.apache.usergrid.persistence.collection.mvcc.MvccLogEntrySerializationStrategy;
import org.apache.usergrid.persistence.collection.serialization.MvccEntitySerializationStrategy;
import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
+import org.apache.usergrid.persistence.collection.serialization.impl.migration.CollectionMigration;
import org.apache.usergrid.persistence.collection.serialization.impl.migration.CollectionMigrationPlugin;
import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
import org.apache.usergrid.persistence.collection.serialization.impl.migration.MvccEntityDataMigrationImpl;
import org.apache.usergrid.persistence.core.guice.ProxyImpl;
-import org.apache.usergrid.persistence.core.migration.data.newimpls.DataMigration2;
-import org.apache.usergrid.persistence.core.migration.data.newimpls.MigrationPlugin;
-import org.apache.usergrid.persistence.core.migration.data.newimpls.MigrationRelationship;
-import org.apache.usergrid.persistence.core.migration.data.newimpls.VersionedMigrationSet;
+import org.apache.usergrid.persistence.core.migration.data.DataMigration;
+import org.apache.usergrid.persistence.core.migration.data.MigrationPlugin;
+import org.apache.usergrid.persistence.core.migration.data.MigrationRelationship;
+import org.apache.usergrid.persistence.core.migration.data.VersionedMigrationSet;
import org.apache.usergrid.persistence.core.migration.schema.Migration;
import com.google.inject.AbstractModule;
import com.google.inject.Inject;
import com.google.inject.Key;
-import com.google.inject.Provider;
import com.google.inject.Provides;
import com.google.inject.Singleton;
import com.google.inject.TypeLiteral;
@@ -67,8 +67,9 @@ public class SerializationModule extends AbstractModule {
//migrations
//we want to make sure our generics are retained, so we use a typeliteral
- Multibinder<DataMigration2<EntityIdScope>> dataMigrationMultibinder =
- Multibinder.newSetBinder( binder(), new TypeLiteral<DataMigration2<EntityIdScope>>() {} );
+ Multibinder<DataMigration<EntityIdScope>> dataMigrationMultibinder =
+ Multibinder.newSetBinder( binder(), new TypeLiteral<DataMigration<EntityIdScope>>() {},
+ CollectionMigration.class );
dataMigrationMultibinder.addBinding().to( MvccEntityDataMigrationImpl.class );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cf32557a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/CollectionMigration.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/CollectionMigration.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/CollectionMigration.java
new file mode 100644
index 0000000..cec4d6f
--- /dev/null
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/CollectionMigration.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence.collection.serialization.impl.migration;/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+import java.lang.annotation.Retention;
+import java.lang.annotation.Target;
+
+import com.google.inject.BindingAnnotation;
+
+import static java.lang.annotation.ElementType.FIELD;
+import static java.lang.annotation.ElementType.METHOD;
+import static java.lang.annotation.ElementType.PARAMETER;
+import static java.lang.annotation.RetentionPolicy.RUNTIME;
+
+
+@BindingAnnotation
+@Target({ FIELD, PARAMETER, METHOD }) @Retention(RUNTIME)
+public @interface CollectionMigration {}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cf32557a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/CollectionMigrationPlugin.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/CollectionMigrationPlugin.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/CollectionMigrationPlugin.java
index 0cf25b2..55e42ff 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/CollectionMigrationPlugin.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/CollectionMigrationPlugin.java
@@ -27,9 +27,9 @@ package org.apache.usergrid.persistence.collection.serialization.impl.migration;
import java.util.Set;
import org.apache.usergrid.persistence.core.migration.data.MigrationInfoSerialization;
-import org.apache.usergrid.persistence.core.migration.data.newimpls.AbstractMigrationPlugin;
-import org.apache.usergrid.persistence.core.migration.data.newimpls.DataMigration2;
-import org.apache.usergrid.persistence.core.migration.data.newimpls.MigrationDataProvider;
+import org.apache.usergrid.persistence.core.migration.data.AbstractMigrationPlugin;
+import org.apache.usergrid.persistence.core.migration.data.DataMigration;
+import org.apache.usergrid.persistence.core.migration.data.MigrationDataProvider;
import com.google.inject.Inject;
import com.google.inject.Singleton;
@@ -46,7 +46,8 @@ public class CollectionMigrationPlugin extends AbstractMigrationPlugin<EntityIdS
@Inject
- public CollectionMigrationPlugin( final Set<DataMigration2<EntityIdScope>> entityDataMigrations,
+ public CollectionMigrationPlugin( @CollectionMigration
+ final Set<DataMigration<EntityIdScope>> entityDataMigrations,
final MigrationDataProvider<EntityIdScope> entityIdScopeDataMigrationProvider,
final MigrationInfoSerialization migrationInfoSerialization ) {
super( entityDataMigrations, entityIdScopeDataMigrationProvider, migrationInfoSerialization );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cf32557a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/MvccEntityDataMigrationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/MvccEntityDataMigrationImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/MvccEntityDataMigrationImpl.java
index 05e5db5..beaaef9 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/MvccEntityDataMigrationImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/MvccEntityDataMigrationImpl.java
@@ -32,7 +32,6 @@ import org.slf4j.LoggerFactory;
import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.collection.EntityVersionCleanupFactory;
import org.apache.usergrid.persistence.collection.MvccEntity;
-import org.apache.usergrid.persistence.collection.impl.EntityDeletedTask;
import org.apache.usergrid.persistence.collection.impl.EntityVersionCleanupTask;
import org.apache.usergrid.persistence.collection.serialization.MvccEntitySerializationStrategy;
import org.apache.usergrid.persistence.collection.serialization.UniqueValue;
@@ -41,11 +40,11 @@ import org.apache.usergrid.persistence.collection.serialization.impl.MvccEntityS
import org.apache.usergrid.persistence.collection.serialization.impl.UniqueValueImpl;
import org.apache.usergrid.persistence.collection.util.EntityUtils;
import org.apache.usergrid.persistence.core.migration.data.DataMigrationException;
-import org.apache.usergrid.persistence.core.migration.data.newimpls.DataMigration2;
-import org.apache.usergrid.persistence.core.migration.data.newimpls.MigrationDataProvider;
-import org.apache.usergrid.persistence.core.migration.data.newimpls.MigrationRelationship;
-import org.apache.usergrid.persistence.core.migration.data.newimpls.ProgressObserver;
-import org.apache.usergrid.persistence.core.migration.data.newimpls.VersionedMigrationSet;
+import org.apache.usergrid.persistence.core.migration.data.DataMigration;
+import org.apache.usergrid.persistence.core.migration.data.MigrationDataProvider;
+import org.apache.usergrid.persistence.core.migration.data.MigrationRelationship;
+import org.apache.usergrid.persistence.core.migration.data.ProgressObserver;
+import org.apache.usergrid.persistence.core.migration.data.VersionedMigrationSet;
import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.field.Field;
@@ -68,7 +67,7 @@ import rx.schedulers.Schedulers;
* Data migration strategy for entities
*/
@Singleton
-public class MvccEntityDataMigrationImpl implements DataMigration2<EntityIdScope> {
+public class MvccEntityDataMigrationImpl implements DataMigration<EntityIdScope> {
private static final Logger LOGGER = LoggerFactory.getLogger( MvccEntityDataMigrationImpl.class );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cf32557a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/guice/TestCollectionModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/guice/TestCollectionModule.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/guice/TestCollectionModule.java
index b42ad94..5718e4f 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/guice/TestCollectionModule.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/guice/TestCollectionModule.java
@@ -20,13 +20,11 @@
package org.apache.usergrid.persistence.collection.guice;
-import java.util.Collections;
-
import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
import org.apache.usergrid.persistence.core.guice.CommonModule;
import org.apache.usergrid.persistence.core.guice.TestModule;
-import org.apache.usergrid.persistence.core.migration.data.newimpls.MigrationDataProvider;
-import org.apache.usergrid.persistence.core.migration.data.newimpls.TestMigrationDataProvider;
+import org.apache.usergrid.persistence.core.migration.data.MigrationDataProvider;
+import org.apache.usergrid.persistence.core.migration.data.TestMigrationDataProvider;
import com.google.inject.TypeLiteral;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cf32557a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV1_3Test.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV1_3Test.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV1_3Test.java
index 5c8a61f..42b10c4 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV1_3Test.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV1_3Test.java
@@ -29,7 +29,6 @@ import org.apache.usergrid.persistence.collection.serialization.MvccEntitySerial
import org.apache.usergrid.persistence.collection.serialization.impl.migration.CollectionMigrationPlugin;
import org.apache.usergrid.persistence.core.guice.ProxyImpl;
import org.apache.usergrid.persistence.core.migration.data.MigrationInfoSerialization;
-import org.apache.usergrid.persistence.core.migration.data.newimpls.VersionedMigrationSet;
import org.apache.usergrid.persistence.core.test.ITRunner;
import org.apache.usergrid.persistence.core.test.UseModules;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cf32557a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/AbstractMvccEntityDataMigrationV1ToV3ImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/AbstractMvccEntityDataMigrationV1ToV3ImplTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/AbstractMvccEntityDataMigrationV1ToV3ImplTest.java
index 9894f07..747ea7b 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/AbstractMvccEntityDataMigrationV1ToV3ImplTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/AbstractMvccEntityDataMigrationV1ToV3ImplTest.java
@@ -33,15 +33,13 @@ import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl;
import org.apache.usergrid.persistence.collection.mvcc.entity.impl.MvccEntityImpl;
import org.apache.usergrid.persistence.collection.serialization.MvccEntitySerializationStrategy;
import org.apache.usergrid.persistence.collection.serialization.impl.CollectionDataVersions;
-import org.apache.usergrid.persistence.collection.serialization.impl.MvccEntitySerializationStrategyV1Impl;
-import org.apache.usergrid.persistence.collection.serialization.impl.MvccEntitySerializationStrategyV3Impl;
import org.apache.usergrid.persistence.core.guice.DataMigrationResetRule;
import org.apache.usergrid.persistence.core.guice.MigrationManagerRule;
import org.apache.usergrid.persistence.core.migration.data.DataMigrationManager;
import org.apache.usergrid.persistence.core.migration.data.TestProgressObserver;
-import org.apache.usergrid.persistence.core.migration.data.newimpls.MigrationDataProvider;
-import org.apache.usergrid.persistence.core.migration.data.newimpls.MigrationRelationship;
-import org.apache.usergrid.persistence.core.migration.data.newimpls.VersionedMigrationSet;
+import org.apache.usergrid.persistence.core.migration.data.MigrationDataProvider;
+import org.apache.usergrid.persistence.core.migration.data.MigrationRelationship;
+import org.apache.usergrid.persistence.core.migration.data.VersionedMigrationSet;
import org.apache.usergrid.persistence.core.test.ITRunner;
import org.apache.usergrid.persistence.core.test.UseModules;
import org.apache.usergrid.persistence.model.entity.Entity;
@@ -150,7 +148,7 @@ public abstract class AbstractMvccEntityDataMigrationV1ToV3ImplTest implements D
final CollectionDataVersions expectedVersion = expectedTargetVersion();
assertEquals( "Correct version returned", newVersion, expectedVersion.getVersion() );
- assertFalse( "Progress observer should not have failed", progressObserver.getFailed() );
+ assertFalse( "Progress observer should not have failed", progressObserver.isFailed() );
assertTrue( "Progress observer should have update messages", progressObserver.getUpdates().size() > 0 );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cf32557a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/MvccEntityDataMigrationV1ToV3ImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/MvccEntityDataMigrationV1ToV3ImplTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/MvccEntityDataMigrationV1ToV3ImplTest.java
index f333fba..7d02b4b 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/MvccEntityDataMigrationV1ToV3ImplTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/MvccEntityDataMigrationV1ToV3ImplTest.java
@@ -20,42 +20,24 @@
package org.apache.usergrid.persistence.collection.serialization.impl.migration;
-import java.util.UUID;
-
import org.junit.Rule;
-import org.junit.Test;
import org.junit.runner.RunWith;
-import org.apache.usergrid.persistence.collection.CollectionScope;
-import org.apache.usergrid.persistence.collection.MvccEntity;
import org.apache.usergrid.persistence.collection.guice.TestCollectionModule;
-import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl;
-import org.apache.usergrid.persistence.collection.mvcc.entity.impl.MvccEntityImpl;
import org.apache.usergrid.persistence.collection.serialization.MvccEntitySerializationStrategy;
import org.apache.usergrid.persistence.collection.serialization.impl.CollectionDataVersions;
import org.apache.usergrid.persistence.collection.serialization.impl.MvccEntitySerializationStrategyV1Impl;
import org.apache.usergrid.persistence.collection.serialization.impl.MvccEntitySerializationStrategyV3Impl;
-import org.apache.usergrid.persistence.core.guice.DataMigrationResetRule;
import org.apache.usergrid.persistence.core.guice.MigrationManagerRule;
import org.apache.usergrid.persistence.core.migration.data.DataMigrationManager;
-import org.apache.usergrid.persistence.core.migration.data.TestProgressObserver;
-import org.apache.usergrid.persistence.core.migration.data.newimpls.MigrationDataProvider;
-import org.apache.usergrid.persistence.core.migration.data.newimpls.MigrationRelationship;
-import org.apache.usergrid.persistence.core.migration.data.newimpls.VersionedMigrationSet;
+import org.apache.usergrid.persistence.core.migration.data.VersionedMigrationSet;
import org.apache.usergrid.persistence.core.test.ITRunner;
import org.apache.usergrid.persistence.core.test.UseModules;
-import org.apache.usergrid.persistence.model.entity.Entity;
-import org.apache.usergrid.persistence.model.entity.Id;
-import org.apache.usergrid.persistence.model.entity.SimpleId;
-import org.apache.usergrid.persistence.model.util.UUIDGenerator;
import com.google.inject.Inject;
-import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
import net.jcip.annotations.NotThreadSafe;
-import rx.Observable;
-
import static org.apache.usergrid.persistence.core.util.IdGenerator.createId;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cf32557a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/MvccEntityDataMigrationV2ToV3ImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/MvccEntityDataMigrationV2ToV3ImplTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/MvccEntityDataMigrationV2ToV3ImplTest.java
index 165105d..1d8c521 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/MvccEntityDataMigrationV2ToV3ImplTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/MvccEntityDataMigrationV2ToV3ImplTest.java
@@ -26,12 +26,11 @@ import org.junit.runner.RunWith;
import org.apache.usergrid.persistence.collection.guice.TestCollectionModule;
import org.apache.usergrid.persistence.collection.serialization.MvccEntitySerializationStrategy;
import org.apache.usergrid.persistence.collection.serialization.impl.CollectionDataVersions;
-import org.apache.usergrid.persistence.collection.serialization.impl.MvccEntitySerializationStrategyV1Impl;
import org.apache.usergrid.persistence.collection.serialization.impl.MvccEntitySerializationStrategyV2Impl;
import org.apache.usergrid.persistence.collection.serialization.impl.MvccEntitySerializationStrategyV3Impl;
import org.apache.usergrid.persistence.core.guice.MigrationManagerRule;
import org.apache.usergrid.persistence.core.migration.data.DataMigrationManager;
-import org.apache.usergrid.persistence.core.migration.data.newimpls.VersionedMigrationSet;
+import org.apache.usergrid.persistence.core.migration.data.VersionedMigrationSet;
import org.apache.usergrid.persistence.core.test.ITRunner;
import org.apache.usergrid.persistence.core.test.UseModules;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cf32557a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/guice/CommonModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/guice/CommonModule.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/guice/CommonModule.java
index 5e61753..bc84b6b 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/guice/CommonModule.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/guice/CommonModule.java
@@ -31,7 +31,7 @@ import org.apache.usergrid.persistence.core.astyanax.CassandraConfigImpl;
import org.apache.usergrid.persistence.core.astyanax.CassandraFig;
import org.apache.usergrid.persistence.core.consistency.TimeService;
import org.apache.usergrid.persistence.core.consistency.TimeServiceImpl;
-import org.apache.usergrid.persistence.core.migration.data.newimpls.MigrationPlugin;
+import org.apache.usergrid.persistence.core.migration.data.MigrationPlugin;
import org.apache.usergrid.persistence.core.migration.schema.Migration;
import org.apache.usergrid.persistence.core.migration.schema.MigrationManager;
import org.apache.usergrid.persistence.core.migration.schema.MigrationManagerFig;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cf32557a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/hystrix/HystrixCassandra.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/hystrix/HystrixCassandra.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/hystrix/HystrixCassandra.java
index 76e0c2d..ab71782 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/hystrix/HystrixCassandra.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/hystrix/HystrixCassandra.java
@@ -30,8 +30,6 @@ import com.netflix.hystrix.HystrixThreadPoolProperties;
/**
* A utility class that creates graph observables wrapped in Hystrix for timeouts and circuit breakers.
- *
- * TODO USERGRId-405 restore this class before merge
*/
public class HystrixCassandra {
@@ -43,32 +41,32 @@ public class HystrixCassandra {
*/
public static final HystrixCommand.Setter
USER_GROUP = HystrixCommand.Setter.withGroupKey( HystrixCommandGroupKey.Factory.asKey( "user" ) ).andThreadPoolPropertiesDefaults(
- HystrixThreadPoolProperties.Setter().withCoreSize( 100 ) );
+ HystrixThreadPoolProperties.Setter().withCoreSize( 1000 ) );
/**
* Command group for asynchronous operations
*/
public static final HystrixCommand.Setter
ASYNC_GROUP = HystrixCommand.Setter.withGroupKey( HystrixCommandGroupKey.Factory.asKey( "async" ) ).andThreadPoolPropertiesDefaults(
- HystrixThreadPoolProperties.Setter().withCoreSize( 50 ) );
+ HystrixThreadPoolProperties.Setter().withCoreSize( 1000 ) );
/**
* Execute an user operation
*/
public static <R> OperationResult<R> user( final Execution<R> execution) {
-// return new HystrixCommand<OperationResult<R>>( USER_GROUP ) {
-//
-// @Override
-// protected OperationResult<R> run() {
+ return new HystrixCommand<OperationResult<R>>( USER_GROUP ) {
+
+ @Override
+ protected OperationResult<R> run() {
try {
return execution.execute();
}
catch ( ConnectionException e ) {
throw new RuntimeException( e );
}
-// }
-// }.execute();
+ }
+ }.execute();
}
@@ -78,18 +76,18 @@ public class HystrixCassandra {
public static <R> OperationResult<R> async( final Execution<R> execution) {
-// return new HystrixCommand<OperationResult<R>>( ASYNC_GROUP ) {
-//
-// @Override
-// protected OperationResult<R> run() {
+ return new HystrixCommand<OperationResult<R>>( ASYNC_GROUP ) {
+
+ @Override
+ protected OperationResult<R> run() {
try {
return execution.execute();
}
catch ( ConnectionException e ) {
throw new RuntimeException( e );
}
-// }
-// }.execute();
+ }
+ }.execute();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cf32557a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/AbstractMigrationPlugin.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/AbstractMigrationPlugin.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/AbstractMigrationPlugin.java
new file mode 100644
index 0000000..1d351d1
--- /dev/null
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/AbstractMigrationPlugin.java
@@ -0,0 +1,128 @@
+/*
+ *
+ * *
+ * * Licensed to the Apache Software Foundation (ASF) under one
+ * * or more contributor license agreements. See the NOTICE file
+ * * distributed with this work for additional information
+ * * regarding copyright ownership. The ASF licenses this file
+ * * to you under the Apache License, Version 2.0 (the
+ * * "License"); you may not use this file except in compliance
+ * * with the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing,
+ * * software distributed under the License is distributed on an
+ * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * * KIND, either express or implied. See the License for the
+ * * specific language governing permissions and limitations
+ * * under the License.
+ * *
+ *
+ */
+
+package org.apache.usergrid.persistence.core.migration.data;
+
+
+import java.util.Set;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/**
+ * Standard implementation logic for plugins to extend
+ * @param <T>
+ */
+public abstract class AbstractMigrationPlugin<T> implements MigrationPlugin {
+
+
+
+ private static final Logger LOG = LoggerFactory.getLogger( AbstractMigrationPlugin.class );
+
+
+ private final Set<DataMigration<T>> entityDataMigrations;
+ private final MigrationDataProvider<T> entityIdScopeDataMigrationProvider;
+ private final MigrationInfoSerialization migrationInfoSerialization;
+
+
+ protected AbstractMigrationPlugin( final Set<DataMigration<T>> entityDataMigrations,
+ final MigrationDataProvider<T> entityIdScopeDataMigrationProvider,
+ final MigrationInfoSerialization migrationInfoSerialization ) {
+ this.entityDataMigrations = entityDataMigrations;
+ this.entityIdScopeDataMigrationProvider = entityIdScopeDataMigrationProvider;
+ this.migrationInfoSerialization = migrationInfoSerialization;
+ }
+
+
+ @Override
+ public void run( final ProgressObserver observer ) {
+
+ //run until complete
+ while(runMigration( observer )){
+ LOG.info( "Migration complete, checking for next run" );
+ }
+
+ }
+
+
+ @Override
+ public int getMaxVersion() {
+
+ int max = 0;
+
+ for(DataMigration<T> entityMigration: entityDataMigrations){
+ max = Math.max( max, entityMigration.getMaxVersion() );
+ }
+
+ return max;
+ }
+
+
+ /**
+ * Try to run the migration
+ *
+ * @return True if we ran a migration
+ */
+ private boolean runMigration( final ProgressObserver po ) {
+ DataMigration<T> migrationToExecute = null;
+
+
+ final int version = migrationInfoSerialization.getVersion( getName() );
+
+ for ( DataMigration<T> entityMigration : entityDataMigrations ) {
+ if ( entityMigration.supports( version ) ) {
+ if ( migrationToExecute != null ) {
+ throw new DataMigrationException(
+ "Two migrations attempted to migration the same version, this is not allowed. Class '"
+ + migrationToExecute.getClass().getName() + "' and class '" + entityMigration
+ .getClass().getName()
+ + "' both support this version. This means something is wired incorrectly" );
+ }
+
+ migrationToExecute = entityMigration;
+ }
+ }
+
+ if(migrationToExecute == null){
+ LOG.info( "No migrations found to execute" );
+ return false;
+ }
+
+ po.start();
+
+
+ //run the migration
+ final int newSystemVersion = migrationToExecute.migrate( version, entityIdScopeDataMigrationProvider, po );
+
+ //write the version
+ migrationInfoSerialization.setVersion( getName(), newSystemVersion );
+
+ po.stop();
+
+ //signal we've run a migration and return
+ return true;
+
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cf32557a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/DataMigration.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/DataMigration.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/DataMigration.java
new file mode 100644
index 0000000..ce250cc
--- /dev/null
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/DataMigration.java
@@ -0,0 +1,58 @@
+/*
+ *
+ * *
+ * * Licensed to the Apache Software Foundation (ASF) under one
+ * * or more contributor license agreements. See the NOTICE file
+ * * distributed with this work for additional information
+ * * regarding copyright ownership. The ASF licenses this file
+ * * to you under the Apache License, Version 2.0 (the
+ * * "License"); you may not use this file except in compliance
+ * * with the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing,
+ * * software distributed under the License is distributed on an
+ * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * * KIND, either express or implied. See the License for the
+ * * specific language governing permissions and limitations
+ * * under the License.
+ * *
+ *
+ */
+
+package org.apache.usergrid.persistence.core.migration.data;
+
+
+/**
+ * Data migration. The internal version to migrate
+ *
+ * @param <T>
+ */
+public interface DataMigration<T> {
+
+ /**
+ * Perform the migration, returning an observable with a single emitted value
+ * @param currentVersion the current version of the system
+ * @param migrationDataProvider
+ * @param observer The observer to receive updates of the progress
+ *
+ * @return The version that the system is now running
+ */
+ public int migrate(final int currentVersion, MigrationDataProvider<T> migrationDataProvider, ProgressObserver observer);
+
+ /**
+ * Check if this version supports migration from the current system version. If this returns false,
+ * migrate will not be invoked
+ * @return
+ */
+ public boolean supports(final int currentVersion);
+
+ /**
+ * Get the max version this migration can migrate to
+ * @return
+ */
+ public int getMaxVersion();
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cf32557a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManagerImpl.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManagerImpl.java
index 41e76e8..7b97e8e 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManagerImpl.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManagerImpl.java
@@ -24,19 +24,13 @@ import java.io.StringWriter;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
-import java.util.concurrent.TimeUnit;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.usergrid.persistence.core.migration.data.newimpls.MigrationPlugin;
-import org.apache.usergrid.persistence.core.migration.data.newimpls.ProgressObserver;
import org.apache.usergrid.persistence.core.migration.schema.MigrationException;
import com.google.common.base.Preconditions;
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheLoader;
-import com.google.common.cache.LoadingCache;
import com.google.inject.Inject;
import com.google.inject.Singleton;
@@ -89,6 +83,8 @@ public class DataMigrationManagerImpl implements DataMigrationManager {
final ProgressObserver observer = new CassandraProgressObserver(plugin.getName());
plugin.run( observer );
+
+
}
@@ -98,6 +94,7 @@ public class DataMigrationManagerImpl implements DataMigrationManager {
@Override
public boolean isRunning() {
+ //we have to get our state from cassandra
for(final String pluginName :getPluginNames()){
if( migrationInfoSerialization.getStatusCode(pluginName) == StatusCode.RUNNING.status){
return true;
@@ -174,6 +171,18 @@ public class DataMigrationManagerImpl implements DataMigrationManager {
@Override
+ public void start() {
+ migrationInfoSerialization.setStatusCode( pluginName, StatusCode.RUNNING.status );
+ }
+
+
+ @Override
+ public void stop() {
+ migrationInfoSerialization.setStatusCode( pluginName, StatusCode.COMPLETE.status );
+ }
+
+
+ @Override
public void failed( final int migrationVersion, final String reason ) {
final String storedMessage = String.format( "Failed to migrate, reason is appended. Error '%s'", reason );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cf32557a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/MigrationDataProvider.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/MigrationDataProvider.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/MigrationDataProvider.java
new file mode 100644
index 0000000..e232d7b
--- /dev/null
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/MigrationDataProvider.java
@@ -0,0 +1,43 @@
+/*
+ *
+ * *
+ * * Licensed to the Apache Software Foundation (ASF) under one
+ * * or more contributor license agreements. See the NOTICE file
+ * * distributed with this work for additional information
+ * * regarding copyright ownership. The ASF licenses this file
+ * * to you under the Apache License, Version 2.0 (the
+ * * "License"); you may not use this file except in compliance
+ * * with the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing,
+ * * software distributed under the License is distributed on an
+ * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * * KIND, either express or implied. See the License for the
+ * * specific language governing permissions and limitations
+ * * under the License.
+ * *
+ *
+ */
+
+package org.apache.usergrid.persistence.core.migration.data;
+
+
+import rx.Observable;
+
+
+/**
+ * An interface for data providers to implement. The migration must take the migrationdata provider as an argument
+ * @param <T>
+ */
+public interface MigrationDataProvider<T> {
+
+
+ /**
+ * Get data that can be used in the migration
+ * @return
+ */
+ public Observable<T> getData();
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cf32557a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/MigrationPlugin.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/MigrationPlugin.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/MigrationPlugin.java
new file mode 100644
index 0000000..897ce27
--- /dev/null
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/MigrationPlugin.java
@@ -0,0 +1,51 @@
+/*
+ *
+ * *
+ * * Licensed to the Apache Software Foundation (ASF) under one
+ * * or more contributor license agreements. See the NOTICE file
+ * * distributed with this work for additional information
+ * * regarding copyright ownership. The ASF licenses this file
+ * * to you under the Apache License, Version 2.0 (the
+ * * "License"); you may not use this file except in compliance
+ * * with the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing,
+ * * software distributed under the License is distributed on an
+ * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * * KIND, either express or implied. See the License for the
+ * * specific language governing permissions and limitations
+ * * under the License.
+ * *
+ *
+ */
+
+package org.apache.usergrid.persistence.core.migration.data;
+
+
+
+/**
+ * A simple interface to return migration plugins. All versions within this migration plugin should have a name
+ */
+public interface MigrationPlugin {
+
+
+ /**
+ * Get the name of the plugin. Must be unique
+ * @return
+ */
+ public String getName();
+
+ /**
+ * Run any migrations that may need to be run in this plugin
+ */
+ public void run(ProgressObserver observer);
+
+
+ /**
+ * Get the max version this plugin supports
+ * @return
+ */
+ int getMaxVersion();
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cf32557a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/MigrationRelationship.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/MigrationRelationship.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/MigrationRelationship.java
new file mode 100644
index 0000000..806768e
--- /dev/null
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/MigrationRelationship.java
@@ -0,0 +1,100 @@
+/*
+ *
+ * *
+ * * Licensed to the Apache Software Foundation (ASF) under one
+ * * or more contributor license agreements. See the NOTICE file
+ * * distributed with this work for additional information
+ * * regarding copyright ownership. The ASF licenses this file
+ * * to you under the Apache License, Version 2.0 (the
+ * * "License"); you may not use this file except in compliance
+ * * with the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing,
+ * * software distributed under the License is distributed on an
+ * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * * KIND, either express or implied. See the License for the
+ * * specific language governing permissions and limitations
+ * * under the License.
+ * *
+ *
+ */
+
+package org.apache.usergrid.persistence.core.migration.data;
+
+
+/**
+ * Simple relationship that defines the current state of the source and destination data versions. Note that
+ * ina current system, the from and then to will be the same instance
+ */
+public class MigrationRelationship<T extends VersionedData> {
+
+ //public so it's FAST. It's also immutable
+
+
+ public final T from;
+ public final T to;
+
+ private final int fromVersion;
+ private final int toVersion;
+
+
+ public MigrationRelationship( T from, T to ) {
+ this.from = from;
+ this.to = to;
+
+ fromVersion = from.getImplementationVersion();
+ toVersion = to.getImplementationVersion();
+ }
+
+
+ /**
+ * Returns true if we need to perform dual writes. IE. the from is not the same as the to
+ * @return
+ */
+ public boolean needsMigration(){
+ return fromVersion != toVersion;
+ }
+
+
+ /**
+ * Return true if this is the migration relationship we should use. The version matches the from
+ * and is <= the to
+ * @param currentVersion
+ * @return
+ */
+ public boolean correctRelationship(final int currentVersion){
+ return currentVersion == fromVersion && currentVersion <= toVersion;
+ }
+
+
+ @Override
+ public boolean equals( final Object o ) {
+ if ( this == o ) {
+ return true;
+ }
+ if ( !( o instanceof MigrationRelationship ) ) {
+ return false;
+ }
+
+ final MigrationRelationship that = ( MigrationRelationship ) o;
+
+ if ( !from.equals( that.from ) ) {
+ return false;
+ }
+ if ( !to.equals( that.to ) ) {
+ return false;
+ }
+
+ return true;
+ }
+
+
+ @Override
+ public int hashCode() {
+ int result = from.hashCode();
+ result = 31 * result + to.hashCode();
+ return result;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cf32557a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/ProgressObserver.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/ProgressObserver.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/ProgressObserver.java
new file mode 100644
index 0000000..50abc6f
--- /dev/null
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/ProgressObserver.java
@@ -0,0 +1,63 @@
+/*
+ *
+ * *
+ * * Licensed to the Apache Software Foundation (ASF) under one
+ * * or more contributor license agreements. See the NOTICE file
+ * * distributed with this work for additional information
+ * * regarding copyright ownership. The ASF licenses this file
+ * * to you under the Apache License, Version 2.0 (the
+ * * "License"); you may not use this file except in compliance
+ * * with the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing,
+ * * software distributed under the License is distributed on an
+ * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * * KIND, either express or implied. See the License for the
+ * * specific language governing permissions and limitations
+ * * under the License.
+ * *
+ *
+ */
+
+package org.apache.usergrid.persistence.core.migration.data;
+
+
+public interface ProgressObserver {
+
+ /**
+ * Signal the process started
+ */
+ public void start();
+
+ /**
+ * Signal the process has stopped
+ */
+ public void stop();
+
+ /**
+ * Mark the migration as failed
+ *
+ * @param migrationVersion The migration version running during the failure
+ * @param reason The reason to save
+ */
+ public void failed( final int migrationVersion, final String reason );
+
+ /**
+ * Mark the migration as failed with a stack trace
+ *
+ * @param migrationVersion The migration version running during the failure
+ * @param reason The error description to save
+ * @param throwable The error that happened
+ */
+ public void failed( final int migrationVersion, final String reason, final Throwable throwable );
+
+
+ /**
+ * Update the status of the migration with the message
+ *
+ * @param message The message to save for the status
+ */
+ public void update( final int migrationVersion, final String message );
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cf32557a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/VersionedData.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/VersionedData.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/VersionedData.java
new file mode 100644
index 0000000..266697a
--- /dev/null
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/VersionedData.java
@@ -0,0 +1,38 @@
+/*
+ *
+ * *
+ * * Licensed to the Apache Software Foundation (ASF) under one
+ * * or more contributor license agreements. See the NOTICE file
+ * * distributed with this work for additional information
+ * * regarding copyright ownership. The ASF licenses this file
+ * * to you under the Apache License, Version 2.0 (the
+ * * "License"); you may not use this file except in compliance
+ * * with the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing,
+ * * software distributed under the License is distributed on an
+ * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * * KIND, either express or implied. See the License for the
+ * * specific language governing permissions and limitations
+ * * under the License.
+ * *
+ *
+ */
+
+package org.apache.usergrid.persistence.core.migration.data;
+
+
+/**
+ * Marker interface for implementations that may migrate their internal format
+ */
+public interface VersionedData {
+
+ /**
+ * Get the version of the implementation
+ * @return
+ */
+ public int getImplementationVersion();
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cf32557a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/VersionedMigrationSet.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/VersionedMigrationSet.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/VersionedMigrationSet.java
new file mode 100644
index 0000000..cadd919
--- /dev/null
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/VersionedMigrationSet.java
@@ -0,0 +1,153 @@
+/*
+ *
+ * *
+ * * Licensed to the Apache Software Foundation (ASF) under one
+ * * or more contributor license agreements. See the NOTICE file
+ * * distributed with this work for additional information
+ * * regarding copyright ownership. The ASF licenses this file
+ * * to you under the Apache License, Version 2.0 (the
+ * * "License"); you may not use this file except in compliance
+ * * with the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing,
+ * * software distributed under the License is distributed on an
+ * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * * KIND, either express or implied. See the License for the
+ * * specific language governing permissions and limitations
+ * * under the License.
+ * *
+ *
+ */
+
+package org.apache.usergrid.persistence.core.migration.data;
+
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import com.google.common.base.Preconditions;
+
+
+/**
+ * A set that represents a set of tuples that are used for
+ * @param <T>
+ */
+public class VersionedMigrationSet<T extends VersionedData> {
+
+
+ /**
+ * Cache so that after our initial lookup, it O(1) since this will be used heavily
+ *
+ */
+ private Map<Integer, MigrationRelationship<T>> cacheVersion = new HashMap<>();
+
+ private List<MigrationRelationship<T>> orderedVersions = new ArrayList<>();
+
+
+ /**
+ * Construct this set from a group of tuples. Imagine the following versions
+ *
+ * v1,
+ * v2,
+ * v3,
+ * v4
+ *
+ * Migrations can jump from v1->v3, but not directly to v4 without an extraneous migration. This would have 2 relationships
+ *
+ * v1, v3
+ * v2, v3
+ * and
+ * v3, v4
+ *
+ *
+ * @param migrations
+ */
+ public VersionedMigrationSet( final MigrationRelationship<T>... migrations ){
+ Preconditions.checkNotNull(migrations, "versions must not be null");
+ Preconditions.checkArgument( migrations.length > 0, "You must specify at least 1 migrationrelationship" );
+
+ orderedVersions.addAll( Arrays.asList(migrations ) );
+
+ Collections.sort( orderedVersions, new VersionedDataComparator() );
+
+ }
+
+
+ /**
+ * Get the migration relationship based on our current version. This will return a range that includes the current
+ * system version as the source, and the highest version we can roll to in the to field
+ * @param currentVersion
+ * @return The MigrationRelationship. Note the from and the to could be the same version in a current system.
+ */
+ public MigrationRelationship<T> getMigrationRelationship( final int currentVersion ){
+
+ final MigrationRelationship<T> relationship = cacheVersion.get( currentVersion );
+
+ if(relationship != null){
+ return relationship;
+ }
+
+ //not there, find it. Not the most efficient, but it happens once per version, which rarely changes, so not a big deal
+
+
+ for(MigrationRelationship<T> current: orderedVersions){
+
+ //not our instance, the from is too high
+ //our from is this instance, so we support this tuple. Our future is >= as well, so we can perform this I/O
+ if ( current.correctRelationship( currentVersion )) {
+ cacheVersion.put( currentVersion, current );
+ return current;
+ }
+
+ }
+
+ //if we get here, something is wrong
+ throw new IllegalArgumentException( "Could not find a migration version for version " + currentVersion + " min found was " + orderedVersions.get( orderedVersions.size()-1 ) );
+
+
+ }
+
+
+ /**
+ * Given the current system version, return the maximum migration version we can move to
+ * @param currentVersion
+ * @return
+ */
+ public int getMaxVersion(final int currentVersion){
+ return getMigrationRelationship( currentVersion ).to.getImplementationVersion();
+ }
+
+
+
+ /**
+ * Orders from high to low
+ */
+ private final class VersionedDataComparator implements Comparator<MigrationRelationship<T>>
+ {
+
+ @Override
+ public int compare( final MigrationRelationship<T> o1, final MigrationRelationship<T> o2 ) {
+ //Put higher target version first, since that's what we want to match based on current state and source
+
+ //order by the source. Put highest source first
+ int compare = Integer.compare( o1.to.getImplementationVersion(), o2.to.getImplementationVersion() ) *-1;
+
+
+ //put higher from first, if we fall within a range here we're good
+ if(compare == 0){
+ compare = Integer.compare( o1.from.getImplementationVersion(), o2.from.getImplementationVersion() ) *-1;
+ }
+
+ return compare;
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cf32557a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/AbstractMigrationPlugin.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/AbstractMigrationPlugin.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/AbstractMigrationPlugin.java
deleted file mode 100644
index 17d88d0..0000000
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/AbstractMigrationPlugin.java
+++ /dev/null
@@ -1,126 +0,0 @@
-/*
- *
- * *
- * * Licensed to the Apache Software Foundation (ASF) under one
- * * or more contributor license agreements. See the NOTICE file
- * * distributed with this work for additional information
- * * regarding copyright ownership. The ASF licenses this file
- * * to you under the Apache License, Version 2.0 (the
- * * "License"); you may not use this file except in compliance
- * * with the License. You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing,
- * * software distributed under the License is distributed on an
- * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * * KIND, either express or implied. See the License for the
- * * specific language governing permissions and limitations
- * * under the License.
- * *
- *
- */
-
-package org.apache.usergrid.persistence.core.migration.data.newimpls;
-
-
-import java.util.Set;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.usergrid.persistence.core.migration.data.DataMigrationException;
-import org.apache.usergrid.persistence.core.migration.data.MigrationInfoSerialization;
-
-
-/**
- * Standard implementation logic for plugins to extend
- * @param <T>
- */
-public abstract class AbstractMigrationPlugin<T> implements MigrationPlugin {
-
-
-
- private static final Logger LOG = LoggerFactory.getLogger( AbstractMigrationPlugin.class );
-
-
- private final Set<DataMigration2<T>> entityDataMigrations;
- private final MigrationDataProvider<T> entityIdScopeDataMigrationProvider;
- private final MigrationInfoSerialization migrationInfoSerialization;
-
-
- protected AbstractMigrationPlugin( final Set<DataMigration2<T>> entityDataMigrations,
- final MigrationDataProvider<T> entityIdScopeDataMigrationProvider,
- final MigrationInfoSerialization migrationInfoSerialization ) {
- this.entityDataMigrations = entityDataMigrations;
- this.entityIdScopeDataMigrationProvider = entityIdScopeDataMigrationProvider;
- this.migrationInfoSerialization = migrationInfoSerialization;
- }
-
-
- @Override
- public void run( final ProgressObserver observer ) {
-
- //run until complete
- while(runMigration( observer )){
- LOG.info( "Migration complete, checking for next run" );
- }
-
- }
-
-
- @Override
- public int getMaxVersion() {
-
- int max = 0;
-
- for(DataMigration2<T> entityMigration: entityDataMigrations){
- max = Math.max( max, entityMigration.getMaxVersion() );
- }
-
- return max;
- }
-
-
- /**
- * Try to run the migration
- *
- * @return True if we ran a migration
- */
- private boolean runMigration( final ProgressObserver po ) {
- DataMigration2<T> migrationToExecute = null;
-
-
- final int version = migrationInfoSerialization.getVersion( getName() );
-
- for ( DataMigration2<T> entityMigration : entityDataMigrations ) {
- if ( entityMigration.supports( version ) ) {
- if ( migrationToExecute != null ) {
- throw new DataMigrationException(
- "Two migrations attempted to migration the same version, this is not allowed. Class '"
- + migrationToExecute.getClass().getName() + "' and class '" + entityMigration
- .getClass().getName()
- + "' both support this version. This means something is wired incorrectly" );
- }
-
- migrationToExecute = entityMigration;
- }
- }
-
- if(migrationToExecute == null){
- LOG.info( "No migrations found to execute" );
- return false;
- }
-
- //run the migration
- final int newSystemVersion = migrationToExecute.migrate( version, entityIdScopeDataMigrationProvider, po );
-
- //write the version
- migrationInfoSerialization.setVersion( getName(), newSystemVersion );
-
- //signal we've run a migration and return
- return true;
-
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cf32557a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/DataMigration2.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/DataMigration2.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/DataMigration2.java
deleted file mode 100644
index 303c11c..0000000
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/DataMigration2.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- *
- * *
- * * Licensed to the Apache Software Foundation (ASF) under one
- * * or more contributor license agreements. See the NOTICE file
- * * distributed with this work for additional information
- * * regarding copyright ownership. The ASF licenses this file
- * * to you under the Apache License, Version 2.0 (the
- * * "License"); you may not use this file except in compliance
- * * with the License. You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing,
- * * software distributed under the License is distributed on an
- * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * * KIND, either express or implied. See the License for the
- * * specific language governing permissions and limitations
- * * under the License.
- * *
- *
- */
-
-package org.apache.usergrid.persistence.core.migration.data.newimpls;
-
-
-/**
- * Data migration. The internal version to migrate
- *
- * @param <T>
- */
-public interface DataMigration2<T> {
-
- /**
- * Perform the migration, returning an observable with a single emitted value
- * @param currentVersion the current version of the system
- * @param migrationDataProvider
- * @param observer The observer to receive updates of the progress
- *
- * @return The version that the system is now running
- */
- public int migrate(final int currentVersion, MigrationDataProvider<T> migrationDataProvider, ProgressObserver observer);
-
- /**
- * Check if this version supports migration from the current system version. If this returns false,
- * migrate will not be invoked
- * @return
- */
- public boolean supports(final int currentVersion);
-
- /**
- * Get the max version this migration can migrate to
- * @return
- */
- public int getMaxVersion();
-
-
-}
[42/50] [abbrv] incubator-usergrid git commit: Merge branch 'pr/190'
into two-dot-o Fixes issue 492
Posted by sf...@apache.org.
Merge branch 'pr/190' into two-dot-o
Fixes issue 492
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/d8f949bf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/d8f949bf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/d8f949bf
Branch: refs/heads/USERGRID-480
Commit: d8f949bf0baa975fab824779ea8677ecd0b26952
Parents: eb0c689 143692f
Author: Dave Johnson <sn...@apache.org>
Authored: Wed Mar 18 18:11:57 2015 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Wed Mar 18 18:11:57 2015 -0400
----------------------------------------------------------------------
.../corepersistence/CpRelationManager.java | 7 ++-
.../results/FilteringLoader.java | 50 +++++++++++++-------
.../corepersistence/StaleIndexCleanupTest.java | 48 +++++++++++++++++++
3 files changed, 86 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
[15/50] [abbrv] incubator-usergrid git commit: Moved back to surefire
4 plugin, this solves the concurrency bug in surefire 47
Posted by sf...@apache.org.
Moved back to surefire 4 plugin, this solves the concurrency bug in surefire 47
http://stackoverflow.com/questions/27225140/intermittent-noclassdeffounderror-when-running-a-maven-surefire-build-in-jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/d3ce4e71
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/d3ce4e71
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/d3ce4e71
Branch: refs/heads/USERGRID-480
Commit: d3ce4e715f43a7a3449e4473afab7ae9d88259d6
Parents: 4812715
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Mar 4 11:52:58 2015 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Mar 4 11:52:58 2015 -0700
----------------------------------------------------------------------
stack/pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d3ce4e71/stack/pom.xml
----------------------------------------------------------------------
diff --git a/stack/pom.xml b/stack/pom.xml
index 3367317..81e7bdd 100644
--- a/stack/pom.xml
+++ b/stack/pom.xml
@@ -1535,7 +1535,7 @@
<dependencies>
<dependency>
<groupId>org.apache.maven.surefire</groupId>
- <artifactId>surefire-junit47</artifactId>
+ <artifactId>surefire-junit4</artifactId>
<version>${surefire.plugin.version}</version>
</dependency>
</dependencies>
[49/50] [abbrv] incubator-usergrid git commit: fix dependency
Posted by sf...@apache.org.
fix dependency
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/70fe2851
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/70fe2851
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/70fe2851
Branch: refs/heads/USERGRID-480
Commit: 70fe28512b20e2c5537c27424f18299a5bb3b58c
Parents: 532b662 8ea46ba
Author: Shawn Feldman <sf...@apache.org>
Authored: Thu Mar 19 14:40:28 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Thu Mar 19 14:40:28 2015 -0600
----------------------------------------------------------------------
.../apache/usergrid/corepersistence/CpEntityManager.java | 3 ---
.../usergrid/corepersistence/CpEntityManagerFactory.java | 9 ++++-----
2 files changed, 4 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/70fe2851/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index b23111a,789e640..a9f00b4
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@@ -2889,9 -2886,10 +2889,6 @@@ public class CpEntityManager implement
// refresh factory indexes
emf.refreshIndex();
--
-- // refresh this Entity Manager's application's index
- EntityIndex ei = managerCache.getEntityIndex( getApplicationScope() );
-- ei.refresh();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/70fe2851/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index 14b7865,f76b9fc..f3d18d0
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@@ -128,7 -126,6 +128,7 @@@ public class CpEntityManagerFactory imp
this.cassandraService = cassandraService;
this.counterUtils = counterUtils;
this.injector = injector;
- this.entityIndex = entityIndex;
++ this.entityIndex = injector.getInstance(EntityIndex.class);
this.managerCache = injector.getInstance( ManagerCache.class );
this.metricsFactory = injector.getInstance( MetricsFactory.class );
@@@ -664,8 -663,10 +664,7 @@@
// refresh special indexes without calling EntityManager refresh because stack overflow
maybeCreateIndexes();
// system app
--
- entityIndex.refresh();
- for ( EntityIndex index : getManagementIndexes() ) {
- index.refresh();
- }
++ entityIndex.refresh();
}
@@@ -675,7 -676,9 +674,7 @@@
return;
}
- entityIndex.initializeIndex();
- for ( EntityIndex index : getManagementIndexes() ) {
- index.initializeIndex();
- }
++// entityIndex.initializeIndex();
}
[47/50] [abbrv] incubator-usergrid git commit: tests passing in
queryindex
Posted by sf...@apache.org.
tests passing in queryindex
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/5f75292c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/5f75292c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/5f75292c
Branch: refs/heads/USERGRID-480
Commit: 5f75292c13fbb8c8acd722ce59c014828f1619c6
Parents: b5ce085
Author: Shawn Feldman <sf...@apache.org>
Authored: Thu Mar 19 13:57:19 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Thu Mar 19 13:57:19 2015 -0600
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 23 +++----
.../corepersistence/CpEntityManagerFactory.java | 2 +-
.../corepersistence/CpManagerCache.java | 6 +-
.../corepersistence/CpRelationManager.java | 19 +++---
.../events/EntityDeletedHandler.java | 3 +-
.../events/EntityVersionDeletedHandler.java | 3 +-
.../results/FilteringLoader.java | 3 +-
.../usergrid/persistence/EntityManager.java | 7 +--
.../cassandra/EntityManagerImpl.java | 9 +--
.../corepersistence/StaleIndexCleanupTest.java | 17 +++--
.../persistence/index/IndexIdentifier.java | 3 +
.../index/impl/EsEntityIndexImpl.java | 66 +++++++++-----------
.../persistence/index/impl/EntityIndexTest.java | 17 +++--
13 files changed, 76 insertions(+), 102 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5f75292c/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 8787410..b23111a 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -36,6 +36,7 @@ import java.util.UUID;
import com.codahale.metrics.Meter;
import org.apache.usergrid.persistence.collection.FieldSet;
import org.apache.usergrid.persistence.core.future.BetterFuture;
+import org.apache.usergrid.persistence.index.ApplicationEntityIndex;
import org.elasticsearch.action.ListenableActionFuture;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -211,6 +212,7 @@ public class CpEntityManager implements EntityManager {
private Timer entGetRepairedEntityTimer;
private Timer updateEntityTimer;
private Meter updateEntityMeter;
+ private EntityIndex ei;
// /** Short-term cache to keep us from reloading same Entity during single request. */
// private LoadingCache<EntityScope, org.apache.usergrid.persistence.model.entity.Entity> entityCache;
@@ -221,7 +223,8 @@ public class CpEntityManager implements EntityManager {
}
@Override
- public void init( EntityManagerFactory emf, UUID applicationId ) {
+ public void init( EntityManagerFactory emf, EntityIndex ei, UUID applicationId ) {
+ this.ei = ei;
Preconditions.checkNotNull( emf, "emf must not be null" );
Preconditions.checkNotNull( applicationId, "applicationId must not be null" );
@@ -286,7 +289,6 @@ public class CpEntityManager implements EntityManager {
@Override
public Health getIndexHealth() {
- EntityIndex ei = managerCache.getEntityIndex( applicationScope );
return ei.getIndexHealth();
}
@@ -1079,7 +1081,7 @@ public class CpEntityManager implements EntityManager {
getCollectionScopeNameFromEntityType( entityRef.getType() ) );
EntityCollectionManager ecm = managerCache.getEntityCollectionManager( collectionScope );
- EntityIndex ei = managerCache.getEntityIndex( getApplicationScope() );
+ ApplicationEntityIndex aei = managerCache.getEntityIndex( getApplicationScope() );
Id entityId = new SimpleId( entityRef.getUuid(), entityRef.getType() );
@@ -1106,7 +1108,7 @@ public class CpEntityManager implements EntityManager {
//Adding graphite metrics
Timer.Context timeESBatch = esDeletePropertyTimer.time();
- BetterFuture future = ei.createBatch().index( defaultIndexScope, cpEntity ).execute();
+ BetterFuture future = aei.createBatch().index( defaultIndexScope, cpEntity ).execute();
timeESBatch.stop();
// update in all containing collections and connection indexes
CpRelationManager rm = ( CpRelationManager ) getRelationManager( entityRef );
@@ -2889,24 +2891,15 @@ public class CpEntityManager implements EntityManager {
emf.refreshIndex();
// refresh this Entity Manager's application's index
- EntityIndex ei = managerCache.getEntityIndex( getApplicationScope() );
ei.refresh();
}
@Override
public void createIndex() {
- EntityIndex ei = managerCache.getEntityIndex( applicationScope );
ei.initializeIndex();
}
- public ListenableActionFuture deleteIndex(){
- EntityIndex ei = managerCache.getEntityIndex( applicationScope );
- return ei.deleteIndex();
- }
-
-
-
@Override
@@ -2981,8 +2974,8 @@ public class CpEntityManager implements EntityManager {
org.apache.usergrid.persistence.model.entity.Entity memberEntity,
String collName ) {
- final EntityIndex ei = getManagerCache().getEntityIndex( getApplicationScope() );
- final EntityIndexBatch batch = ei.createBatch();
+ final ApplicationEntityIndex aie = getManagerCache().getEntityIndex( getApplicationScope() );
+ final EntityIndexBatch batch = aie.createBatch();
// index member into entity collection | type scope
IndexScope collectionIndexScope = new IndexScopeImpl( collectionEntity.getId(),
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5f75292c/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index e70e538..14b7865 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -199,7 +199,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
private EntityManager _getEntityManager( UUID applicationId ) {
EntityManager em = new CpEntityManager();
- em.init( this, applicationId );
+ em.init( this,entityIndex ,applicationId );
return em;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5f75292c/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpManagerCache.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpManagerCache.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpManagerCache.java
index aa8a139..4e35f8c 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpManagerCache.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpManagerCache.java
@@ -25,6 +25,7 @@ import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.graph.GraphManager;
import org.apache.usergrid.persistence.graph.GraphManagerFactory;
+import org.apache.usergrid.persistence.index.ApplicationEntityIndex;
import org.apache.usergrid.persistence.index.EntityIndex;
import org.apache.usergrid.persistence.index.EntityIndexFactory;
import org.apache.usergrid.persistence.map.MapManager;
@@ -62,16 +63,15 @@ public class CpManagerCache implements ManagerCache {
@Override
public EntityCollectionManager getEntityCollectionManager( CollectionScope scope ) {
-
//cache is now in the colletion manager level
return ecmf.createCollectionManager( scope );
}
@Override
- public EntityIndex getEntityIndex( ApplicationScope appScope ) {
+ public ApplicationEntityIndex getEntityIndex( ApplicationScope appScope ) {
- return eif.createEntityIndex( appScope );
+ return eif.createApplicationEntityIndex( appScope );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5f75292c/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index 2eeee28..df01c47 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -30,6 +30,7 @@ import java.util.Set;
import java.util.UUID;
import org.apache.usergrid.persistence.core.future.BetterFuture;
+import org.apache.usergrid.persistence.index.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.Assert;
@@ -75,10 +76,6 @@ import org.apache.usergrid.persistence.graph.impl.SimpleEdge;
import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdge;
import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdgeType;
import org.apache.usergrid.persistence.graph.impl.SimpleSearchEdgeType;
-import org.apache.usergrid.persistence.index.EntityIndex;
-import org.apache.usergrid.persistence.index.EntityIndexBatch;
-import org.apache.usergrid.persistence.index.IndexScope;
-import org.apache.usergrid.persistence.index.SearchTypes;
import org.apache.usergrid.persistence.index.impl.IndexScopeImpl;
import org.apache.usergrid.persistence.index.query.CandidateResult;
import org.apache.usergrid.persistence.index.query.CandidateResults;
@@ -397,7 +394,7 @@ public class CpRelationManager implements RelationManager {
// loop through all types of edge to target
- final EntityIndex ei = managerCache.getEntityIndex( applicationScope );
+ final ApplicationEntityIndex ei = managerCache.getEntityIndex(applicationScope);
final EntityIndexBatch entityIndexBatch = ei.createBatch();
@@ -828,7 +825,7 @@ public class CpRelationManager implements RelationManager {
org.apache.usergrid.persistence.model.entity.Entity memberEntity =
((CpEntityManager)em).load( new CpEntityManager.EntityScope( memberScope, entityId));
- final EntityIndex ei = managerCache.getEntityIndex( applicationScope );
+ final ApplicationEntityIndex ei = managerCache.getEntityIndex( applicationScope );
final EntityIndexBatch batch = ei.createBatch();
// remove item from collection index
@@ -945,7 +942,7 @@ public class CpRelationManager implements RelationManager {
cpHeadEntity.getId(),
CpNamingUtils.getCollectionScopeNameFromCollectionName( collName ) );
- final EntityIndex ei = managerCache.getEntityIndex( applicationScope );
+ final ApplicationEntityIndex ei = managerCache.getEntityIndex( applicationScope );
final SearchTypes types = SearchTypes.fromTypes( collection.getType() );
@@ -1060,7 +1057,7 @@ public class CpRelationManager implements RelationManager {
GraphManager gm = managerCache.getGraphManager( applicationScope );
gm.writeEdge( edge ).toBlockingObservable().last();
- EntityIndex ei = managerCache.getEntityIndex( applicationScope );
+ ApplicationEntityIndex ei = managerCache.getEntityIndex( applicationScope );
EntityIndexBatch batch = ei.createBatch();
// Index the new connection in app|source|type context
@@ -1292,7 +1289,7 @@ public class CpRelationManager implements RelationManager {
GraphManager gm = managerCache.getGraphManager( applicationScope );
gm.deleteEdge( edge ).toBlockingObservable().last();
- final EntityIndex ei = managerCache.getEntityIndex( applicationScope );
+ final ApplicationEntityIndex ei = managerCache.getEntityIndex( applicationScope );
final EntityIndexBatch batch = ei.createBatch();
// Deindex the connection in app|source|type context
@@ -1372,7 +1369,7 @@ public class CpRelationManager implements RelationManager {
final SearchTypes searchTypes = SearchTypes.fromNullableTypes( connectedEntityType );
- final EntityIndex ei = managerCache.getEntityIndex( applicationScope );
+ final ApplicationEntityIndex ei = managerCache.getEntityIndex( applicationScope );
logger.debug("Searching connected entities from scope {}:{}",
@@ -1465,7 +1462,7 @@ public class CpRelationManager implements RelationManager {
final SearchTypes searchTypes = SearchTypes.fromNullableTypes( query.getEntityType() );
- EntityIndex ei = managerCache.getEntityIndex( applicationScope );
+ ApplicationEntityIndex ei = managerCache.getEntityIndex( applicationScope );
logger.debug( "Searching connections from the scope {}:{} with types {}", new Object[] {
indexScope.getOwner().toString(), indexScope.getName(), searchTypes
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5f75292c/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityDeletedHandler.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityDeletedHandler.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityDeletedHandler.java
index 94ef5e4..85c59da 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityDeletedHandler.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityDeletedHandler.java
@@ -20,6 +20,7 @@ package org.apache.usergrid.corepersistence.events;
import java.util.UUID;
+import org.apache.usergrid.persistence.index.ApplicationEntityIndex;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -70,7 +71,7 @@ public class EntityDeletedHandler implements EntityDeleted {
} );
CpEntityManagerFactory cpemf = ( CpEntityManagerFactory ) emf;
- final EntityIndex ei = cpemf.getManagerCache().getEntityIndex( scope );
+ final ApplicationEntityIndex ei = cpemf.getManagerCache().getEntityIndex( scope );
final IndexScope indexScope =
new IndexScopeImpl( new SimpleId( scope.getOwner().getUuid(), scope.getOwner().getType() ),
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5f75292c/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java
index c45949b..ece7562 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java
@@ -29,6 +29,7 @@ import org.apache.usergrid.persistence.collection.MvccEntity;
import org.apache.usergrid.persistence.collection.MvccLogEntry;
import org.apache.usergrid.persistence.collection.event.EntityVersionDeleted;
import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
+import org.apache.usergrid.persistence.index.ApplicationEntityIndex;
import org.apache.usergrid.persistence.index.EntityIndex;
import org.apache.usergrid.persistence.index.EntityIndexBatch;
import org.apache.usergrid.persistence.index.IndexScope;
@@ -85,7 +86,7 @@ public class EntityVersionDeletedHandler implements EntityVersionDeleted {
CpEntityManagerFactory cpemf = (CpEntityManagerFactory)emf;
- final EntityIndex ei = cpemf.getManagerCache().getEntityIndex( scope );
+ final ApplicationEntityIndex ei = cpemf.getManagerCache().getEntityIndex( scope );
final IndexScope indexScope = new IndexScopeImpl(
new SimpleId(scope.getOwner().getUuid(), scope.getOwner().getType()),
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5f75292c/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java
index b571f21..0a0f71a 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java
@@ -30,6 +30,7 @@ import java.util.UUID;
import javax.annotation.Nullable;
+import org.apache.usergrid.persistence.index.ApplicationEntityIndex;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -82,7 +83,7 @@ public class FilteringLoader implements ResultsLoader {
this.applicationScope = applicationScope;
this.indexScope = indexScope;
- final EntityIndex index = managerCache.getEntityIndex( applicationScope );
+ final ApplicationEntityIndex index = managerCache.getEntityIndex( applicationScope );
indexBatch = index.createBatch();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5f75292c/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
index 2a951aa..71dcd07 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
@@ -17,6 +17,7 @@
package org.apache.usergrid.persistence;
+import org.apache.usergrid.persistence.index.EntityIndex;
import org.apache.usergrid.persistence.index.query.Query;
import java.nio.ByteBuffer;
import java.util.Collection;
@@ -694,12 +695,8 @@ public interface EntityManager {
*/
void createIndex();
- /**
- * Create the index, should ONLY ever be called the first time an application is created
- */
- ListenableActionFuture deleteIndex();
- public void init( EntityManagerFactory emf, UUID applicationId);
+ public void init( EntityManagerFactory emf, EntityIndex entityIndex, UUID applicationId);
/** For testing purposes */
public void flushManagerCaches();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5f75292c/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
index 966a601..803512b 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
@@ -36,6 +36,7 @@ import java.util.UUID;
import javax.annotation.Resource;
+import org.apache.usergrid.persistence.index.EntityIndex;
import org.elasticsearch.action.ListenableActionFuture;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -214,7 +215,7 @@ public class EntityManagerImpl implements EntityManager {
@Override
- public void init(EntityManagerFactory emf, UUID applicationId) {
+ public void init(EntityManagerFactory emf, EntityIndex entityIndex, UUID applicationId) {
init( (EntityManagerFactoryImpl)emf, null, null, applicationId, false);
}
@@ -2895,12 +2896,6 @@ public class EntityManagerImpl implements EntityManager {
}
- @Override
- public ListenableActionFuture deleteIndex() {
- //no op
- return null;
- }
-
@Override
public EntityRef getGroupRoleRef( UUID ownerId, String roleName) throws Exception {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5f75292c/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
index 1becee4..fdd230c 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
@@ -24,6 +24,7 @@ import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
+import org.apache.usergrid.persistence.index.*;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -46,10 +47,6 @@ import org.apache.usergrid.persistence.collection.EntityCollectionManager;
import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
-import org.apache.usergrid.persistence.index.EntityIndex;
-import org.apache.usergrid.persistence.index.EntityIndexFactory;
-import org.apache.usergrid.persistence.index.IndexScope;
-import org.apache.usergrid.persistence.index.SearchTypes;
import org.apache.usergrid.persistence.index.impl.IndexScopeImpl;
import org.apache.usergrid.persistence.index.query.CandidateResults;
import org.apache.usergrid.persistence.index.query.Query;
@@ -155,12 +152,12 @@ public class StaleIndexCleanupTest extends AbstractCoreIT {
UUID newVersion = getCpEntity( thing ).getVersion();
- assertEquals( 2, queryCollectionCp( "things", "thing", "select * order by ordinal desc" ).size() );
+ assertEquals(2, queryCollectionCp("things", "thing", "select * order by ordinal desc").size());
//now run enable events and ensure we clean up
- System.setProperty( EVENTS_DISABLED, "false" );
+ System.setProperty(EVENTS_DISABLED, "false");
- final Results results = queryCollectionEm( "things", "select * order by ordinal desc" );
+ final Results results = queryCollectionEm("things", "select * order by ordinal desc");
assertEquals( 1, results.size() );
assertEquals(1, results.getEntities().get( 0 ).getProperty( "ordinal" ));
@@ -169,9 +166,9 @@ public class StaleIndexCleanupTest extends AbstractCoreIT {
//ensure it's actually gone
final CandidateResults candidates = queryCollectionCp( "things", "thing", "select * order by ordinal desc" );
- assertEquals( 1, candidates.size() );
+ assertEquals(1, candidates.size());
- assertEquals(newVersion, candidates.get( 0 ).getVersion());
+ assertEquals(newVersion, candidates.get(0).getVersion());
}
@@ -483,7 +480,7 @@ public class StaleIndexCleanupTest extends AbstractCoreIT {
ApplicationScope as = new ApplicationScopeImpl(
new SimpleId( em.getApplicationId(), TYPE_APPLICATION ) );
- EntityIndex ei = eif.createEntityIndex( as );
+ ApplicationEntityIndex ei = eif.createApplicationEntityIndex(as);
IndexScope is = new IndexScopeImpl( new SimpleId( em.getApplicationId(), TYPE_APPLICATION ),
CpNamingUtils.getCollectionScopeNameFromCollectionName( collName ) );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5f75292c/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java
index 1ac9d49..c659ed6 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java
@@ -20,12 +20,15 @@
package org.apache.usergrid.persistence.index;
+import com.google.inject.Inject;
+
/**
* Class is used to generate an index name and alias name
*/
public class IndexIdentifier{
private final IndexFig config;
+ @Inject
public IndexIdentifier(IndexFig config) {
this.config = config;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5f75292c/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index d510a8d..e92e133 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -55,6 +55,8 @@ import org.elasticsearch.index.query.*;
import org.elasticsearch.indices.IndexAlreadyExistsException;
import org.elasticsearch.indices.IndexMissingException;
+import org.elasticsearch.indices.InvalidAliasNameException;
+import org.elasticsearch.rest.action.admin.indices.alias.delete.AliasesMissingException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -119,9 +121,6 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
this.esProvider = provider;
this.config = config;
-
-
-
this.alias = indexIdentifier.getAlias();
this.aliasCache = indexCache;
this.addTimer = metricsFactory
@@ -143,14 +142,14 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
if(indexes == null || indexes.length==0) {
addIndex(null, numberOfShards, numberOfReplicas, config.getWriteConsistencyLevel());
}
+
}
@Override
public void addIndex(final String indexSuffix,final int numberOfShards, final int numberOfReplicas, final String writeConsistency) {
- String normalizedSuffix = StringUtils.isNotEmpty(indexSuffix) ? indexSuffix : null;
try {
//get index name with suffix attached
- String indexName = indexIdentifier.getIndex(normalizedSuffix);
+ String indexName = indexIdentifier.getIndex(indexSuffix);
//Create index
try {
@@ -186,12 +185,10 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
//We do NOT want to create an alias if the index already exists, we'll overwrite the indexes that
//may have been set via other administrative endpoint
- addAlias(normalizedSuffix);
+ addAlias(indexSuffix);
testNewIndex();
-
-
} catch (IndexAlreadyExistsException expected) {
// this is expected to happen if index already exists, it's a no-op and swallow
} catch (IOException e) {
@@ -202,51 +199,47 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
@Override
public void addAlias(final String indexSuffix) {
-
- final Timer.Context timeRemoveAlias = updateAliasTimer.time();
+ Timer.Context timer = updateAliasTimer.time();
try {
-
-
+ Boolean isAck;
String indexName = indexIdentifier.getIndex(indexSuffix);
final AdminClient adminClient = esProvider.getClient().admin();
- String[] indexNames = getIndexesFromEs(AliasType.Write);
-
-
- final IndicesAliasesRequestBuilder aliasesRequestBuilder = adminClient.indices().prepareAliases();
+ String[] indexNames = getIndexes(AliasType.Write);
- //remove the write alias from it's target
+ int count = 0;
+ IndicesAliasesRequestBuilder aliasesRequestBuilder = adminClient.indices().prepareAliases();
for ( String currentIndex : indexNames ) {
aliasesRequestBuilder.removeAlias( currentIndex, alias.getWriteAlias() );
- logger.info("Removing existing write Alias Name [{}] from Index [{}]", alias.getWriteAlias(), currentIndex);
+ count++;
}
+ if(count>0) {
+ isAck = aliasesRequestBuilder.execute().actionGet().isAcknowledged();
+ logger.info("Removed Index Name from Alias=[{}] ACK=[{}]", alias, isAck);
+ }
+ aliasesRequestBuilder = adminClient.indices().prepareAliases();
//Added For Graphite Metrics
-
// add read alias
- aliasesRequestBuilder.addAlias( indexName, alias.getReadAlias());
- logger.info("Created new read Alias Name [{}] on Index [{}]", alias.getReadAlias(), indexName);
-
+ aliasesRequestBuilder.addAlias(
+ indexName, alias.getReadAlias());
+ //Added For Graphite Metrics
//add write alias
- aliasesRequestBuilder.addAlias( indexName, alias.getWriteAlias() );
-
- logger.info("Created new write Alias Name [{}] on Index [{}]", alias.getWriteAlias(), indexName);
+ aliasesRequestBuilder.addAlias(
+ indexName, alias.getWriteAlias());
- final IndicesAliasesResponse result = aliasesRequestBuilder.execute().actionGet();
+ isAck = aliasesRequestBuilder.execute().actionGet().isAcknowledged();
- final boolean isAcknowledged = result.isAcknowledged();
+ logger.info("Created new aliases ACK=[{}]", isAck);
- if(!isAcknowledged){
- throw new RuntimeException( "Unable to add aliases to the new index. Elasticsearch did not acknowledge to the alias change for index '" + indexSuffix + "'");
- }
+ aliasCache.invalidate(alias);
+ } catch (Exception e) {
+ logger.warn("Failed to create alias ", e);
}
- finally{
- //invalidate the alias
- aliasCache.invalidate(alias);
- //stop the timer
- timeRemoveAlias.stop();
+ finally {
+ timer.stop();
}
}
@@ -255,7 +248,6 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
return aliasCache.getIndexes(alias, aliasType);
}
-
/**
* Get our index info from ES, but clear our cache first
* @param aliasType
@@ -266,8 +258,6 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
return getIndexes( aliasType );
}
-
-
/**
* Tests writing a document to a new index to ensure it's working correctly. See this post:
* http://s.apache.org/index-missing-exception
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5f75292c/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
index 483efb9..28c548d 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
@@ -81,22 +81,21 @@ public class EntityIndexTest extends BaseIT {
@Test
public void testIndex() throws IOException, InterruptedException {
- Id appId = new SimpleId( "application" );
+ Id appId = new SimpleId("application");
+ ei.initializeIndex();
- ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
+ ApplicationScope applicationScope = new ApplicationScopeImpl(appId);
ApplicationEntityIndex entityIndex = eif.createApplicationEntityIndex(applicationScope);
-
final String entityType = "thing";
- IndexScope indexScope = new IndexScopeImpl( appId, "things" );
- final SearchTypes searchTypes = SearchTypes.fromTypes( entityType );
+ IndexScope indexScope = new IndexScopeImpl(appId, "things");
+ final SearchTypes searchTypes = SearchTypes.fromTypes(entityType);
- insertJsonBlob(entityIndex, entityType, indexScope, "/sample-large.json",101,0);
-
- ei.refresh();
+ insertJsonBlob(entityIndex, entityType, indexScope, "/sample-large.json", 101, 0);
+ ei.refresh();
- testQueries( indexScope, searchTypes, entityIndex );
+ testQueries(indexScope, searchTypes, entityIndex);
}
@Test
[18/50] [abbrv] incubator-usergrid git commit: Upgraded plexus utils
and fixes build concurrency issues with surefire
Posted by sf...@apache.org.
Upgraded plexus utils and fixes build concurrency issues with surefire
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/e54b2639
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/e54b2639
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/e54b2639
Branch: refs/heads/USERGRID-480
Commit: e54b2639a10cc87cf7eba9a69c75614020f6cdb3
Parents: 08719b7
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Mar 4 15:57:11 2015 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Mar 4 15:57:11 2015 -0700
----------------------------------------------------------------------
stack/core/pom.xml | 7 +++++-
.../core/metrics/MetricsFactoryImpl.java | 3 +--
.../persistence/index/guice/IndexModule.java | 7 +++---
.../index/impl/EsEntityIndexFactoryImpl.java | 9 ++++++--
stack/pom.xml | 24 +++++++++++++-------
stack/rest/pom.xml | 2 +-
stack/services/pom.xml | 7 +++++-
.../usergrid/management/OrganizationIT.java | 2 +-
8 files changed, 42 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e54b2639/stack/core/pom.xml
----------------------------------------------------------------------
diff --git a/stack/core/pom.xml b/stack/core/pom.xml
index 77ce82e..d30bedb 100644
--- a/stack/core/pom.xml
+++ b/stack/core/pom.xml
@@ -101,9 +101,14 @@
<dependencies>
<dependency>
<groupId>org.apache.maven.surefire</groupId>
- <artifactId>surefire-junit47</artifactId>
+ <artifactId>${surefire.plugin.artifactName}</artifactId>
<version>${surefire.plugin.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.codehaus.plexus</groupId>
+ <artifactId>plexus-utils</artifactId>
+ <version>3.0.21</version>
+ </dependency>
</dependencies>
</plugin>
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e54b2639/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/metrics/MetricsFactoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/metrics/MetricsFactoryImpl.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/metrics/MetricsFactoryImpl.java
index 2528070..773828d 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/metrics/MetricsFactoryImpl.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/metrics/MetricsFactoryImpl.java
@@ -35,7 +35,7 @@ import java.util.concurrent.TimeUnit;
@Singleton
public class MetricsFactoryImpl implements MetricsFactory {
- private final MetricsFig metricsFig;
+
private MetricRegistry registry;
private GraphiteReporter graphiteReporter;
private JmxReporter jmxReporter;
@@ -44,7 +44,6 @@ public class MetricsFactoryImpl implements MetricsFactory {
@Inject
public MetricsFactoryImpl(MetricsFig metricsFig) {
- this.metricsFig = metricsFig;
registry = new MetricRegistry();
String metricsHost = metricsFig.getHost();
if(!metricsHost.equals("false")) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e54b2639/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
index d9a14c9..6fee17e 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
@@ -22,6 +22,8 @@ package org.apache.usergrid.persistence.index.guice;
import org.apache.usergrid.persistence.index.*;
import com.google.inject.AbstractModule;
import com.google.inject.assistedinject.FactoryModuleBuilder;
+
+import org.apache.usergrid.persistence.index.impl.EsEntityIndexFactoryImpl;
import org.apache.usergrid.persistence.index.impl.EsEntityIndexImpl;
import org.apache.usergrid.persistence.index.impl.EsIndexBufferConsumerImpl;
import org.apache.usergrid.persistence.index.impl.EsIndexBufferProducerImpl;
@@ -36,9 +38,8 @@ public class IndexModule extends AbstractModule {
// install our configuration
install(new GuicyFigModule(IndexFig.class));
- install(new FactoryModuleBuilder()
- .implement(EntityIndex.class, EsEntityIndexImpl.class)
- .build(EntityIndexFactory.class));
+
+ bind(EntityIndexFactory.class).to( EsEntityIndexFactoryImpl.class );
bind(IndexBufferProducer.class).to(EsIndexBufferProducerImpl.class);
bind(IndexBufferConsumer.class).to(EsIndexBufferConsumerImpl.class).asEagerSingleton();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e54b2639/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexFactoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexFactoryImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexFactoryImpl.java
index 0c643a1..4bf8b59 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexFactoryImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexFactoryImpl.java
@@ -27,6 +27,7 @@ import com.google.inject.assistedinject.Assisted;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.index.EntityIndex;
import org.apache.usergrid.persistence.index.EntityIndexFactory;
+import org.apache.usergrid.persistence.index.IndexBufferProducer;
import org.apache.usergrid.persistence.index.IndexFig;
import java.util.concurrent.ExecutionException;
@@ -39,18 +40,22 @@ public class EsEntityIndexFactoryImpl implements EntityIndexFactory{
private final IndexFig config;
private final EsProvider provider;
private final EsIndexCache indexCache;
+ private final IndexBufferProducer indexBatchBufferProducer;
private LoadingCache<ApplicationScope, EntityIndex> eiCache =
CacheBuilder.newBuilder().maximumSize( 1000 ).build( new CacheLoader<ApplicationScope, EntityIndex>() {
public EntityIndex load( ApplicationScope scope ) {
- return new EsEntityIndexImpl(scope,config,provider,indexCache);
+ return new EsEntityIndexImpl(scope,config, indexBatchBufferProducer, provider,indexCache);
}
} );
+
@Inject
- public EsEntityIndexFactoryImpl(final IndexFig config, final EsProvider provider, final EsIndexCache indexCache){
+ public EsEntityIndexFactoryImpl( final IndexFig config, final EsProvider provider, final EsIndexCache indexCache,
+ final IndexBufferProducer indexBatchBufferProducer ){
this.config = config;
this.provider = provider;
this.indexCache = indexCache;
+ this.indexBatchBufferProducer = indexBatchBufferProducer;
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e54b2639/stack/pom.xml
----------------------------------------------------------------------
diff --git a/stack/pom.xml b/stack/pom.xml
index 4c0dcd7..f71c029 100644
--- a/stack/pom.xml
+++ b/stack/pom.xml
@@ -127,6 +127,7 @@
<metrics.version>3.0.0</metrics.version>
<rx.version>0.19.6</rx.version>
+ <surefire.plugin.artifactName>surefire-junit47</surefire.plugin.artifactName>
<surefire.plugin.version>2.18.1</surefire.plugin.version>
<powermock.version>1.6.1</powermock.version>
@@ -1503,8 +1504,8 @@
<forkCount>${usergrid.it.forkCount}</forkCount>
<reuseForks>${usergrid.it.reuseForks}</reuseForks>
<threadCount>${usergrid.it.forkCount}</threadCount>
- <argLine>-Xmx${ug.heapmax} -Xms${ug.heapmin} -javaagent:${settings.localRepository}/com/github/stephenc/jamm/0.2.5/jamm-0.2.5.jar ${ug.argline}
- </argLine>
+ <argLine>-Xmx${ug.heapmax} -Xms${ug.heapmin} -javaagent:${settings.localRepository}/com/github/stephenc/jamm/0.2.5/jamm-0.2.5.jar ${ug.argline}</argLine>
+ <testFailureIgnore>false</testFailureIgnore>
</configuration>
<!-- TODO, we may need an exclusion. Appears to be a classloader bug
@@ -1515,12 +1516,19 @@
<groupId>org.apache.maven.surefire</groupId>
<artifactId>surefire-junit4</artifactId>
<version>${surefire.plugin.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.apache.maven.surfire</groupId>
- <artifactId>common-junit3</artifactId>
- </exclusion>
- </exclusions>
+
+ <!--<exclusions>-->
+ <!--<exclusion>-->
+ <!--<groupId>org.apache.maven.surfire</groupId>-->
+ <!--<artifactId>common-junit3</artifactId>-->
+ <!--</exclusion>-->
+ <!--</exclusions>-->
+ </dependency>
+ <!-- override plex utils, otherwise bug from above SO post happens-->
+ <dependency>
+ <groupId>org.codehaus.plexus</groupId>
+ <artifactId>plexus-utils</artifactId>
+ <version>3.0.21</version>
</dependency>
</dependencies>
</plugin>
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e54b2639/stack/rest/pom.xml
----------------------------------------------------------------------
diff --git a/stack/rest/pom.xml b/stack/rest/pom.xml
index 3ad58b4..5a856de 100644
--- a/stack/rest/pom.xml
+++ b/stack/rest/pom.xml
@@ -115,7 +115,7 @@
<dependencies>
<dependency>
<groupId>org.apache.maven.surefire</groupId>
- <artifactId>surefire-junit47</artifactId>
+ <artifactId>${surefire.plugin.artifactName}</artifactId>
<version>${surefire.plugin.version}</version>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e54b2639/stack/services/pom.xml
----------------------------------------------------------------------
diff --git a/stack/services/pom.xml b/stack/services/pom.xml
index 46b0878..f5e8dd5 100644
--- a/stack/services/pom.xml
+++ b/stack/services/pom.xml
@@ -108,9 +108,14 @@
<dependencies>
<dependency>
<groupId>org.apache.maven.surefire</groupId>
- <artifactId>surefire-junit47</artifactId>
+ <artifactId>${surefire.plugin.artifactName}</artifactId>
<version>${surefire.plugin.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.codehaus.plexus</groupId>
+ <artifactId>plexus-utils</artifactId>
+ <version>3.0.21</version>
+ </dependency>
</dependencies>
</plugin>
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e54b2639/stack/services/src/test/java/org/apache/usergrid/management/OrganizationIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/management/OrganizationIT.java b/stack/services/src/test/java/org/apache/usergrid/management/OrganizationIT.java
index cc89122..5bf0c3d 100644
--- a/stack/services/src/test/java/org/apache/usergrid/management/OrganizationIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/management/OrganizationIT.java
@@ -90,7 +90,7 @@ public class OrganizationIT {
OrganizationInfo organization2 = setup.getMgmtSvc().getOrganizationForApplication( applicationId );
assertNotNull( organization2 );
- assertEquals( "wrong organization name", organization.getName(), organization2.getName() );
+ assertEquals( "wrong organization name", organization.getOrganization().getName(), organization2.getName() );
boolean verified = setup.getMgmtSvc().verifyAdminUserPassword( organization.getOwner().getUuid(), "test" );
assertTrue( verified );
[21/50] [abbrv] incubator-usergrid git commit: Removed Hystrix for
performance testing.
Posted by sf...@apache.org.
Removed Hystrix for performance testing.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/2e77c631
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/2e77c631
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/2e77c631
Branch: refs/heads/USERGRID-480
Commit: 2e77c631683168befdf377283f0374576ab52a3a
Parents: e83caf2
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Mar 4 19:21:10 2015 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Mar 4 19:21:10 2015 -0700
----------------------------------------------------------------------
.../core/hystrix/HystrixCassandra.java | 26 +++++++++++---------
1 file changed, 14 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2e77c631/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/hystrix/HystrixCassandra.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/hystrix/HystrixCassandra.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/hystrix/HystrixCassandra.java
index 7d5316f..76e0c2d 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/hystrix/HystrixCassandra.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/hystrix/HystrixCassandra.java
@@ -30,6 +30,8 @@ import com.netflix.hystrix.HystrixThreadPoolProperties;
/**
* A utility class that creates graph observables wrapped in Hystrix for timeouts and circuit breakers.
+ *
+ * TODO USERGRId-405 restore this class before merge
*/
public class HystrixCassandra {
@@ -55,18 +57,18 @@ public class HystrixCassandra {
* Execute an user operation
*/
public static <R> OperationResult<R> user( final Execution<R> execution) {
- return new HystrixCommand<OperationResult<R>>( USER_GROUP ) {
-
- @Override
- protected OperationResult<R> run() {
+// return new HystrixCommand<OperationResult<R>>( USER_GROUP ) {
+//
+// @Override
+// protected OperationResult<R> run() {
try {
return execution.execute();
}
catch ( ConnectionException e ) {
throw new RuntimeException( e );
}
- }
- }.execute();
+// }
+// }.execute();
}
@@ -76,18 +78,18 @@ public class HystrixCassandra {
public static <R> OperationResult<R> async( final Execution<R> execution) {
- return new HystrixCommand<OperationResult<R>>( ASYNC_GROUP ) {
-
- @Override
- protected OperationResult<R> run() {
+// return new HystrixCommand<OperationResult<R>>( ASYNC_GROUP ) {
+//
+// @Override
+// protected OperationResult<R> run() {
try {
return execution.execute();
}
catch ( ConnectionException e ) {
throw new RuntimeException( e );
}
- }
- }.execute();
+// }
+// }.execute();
}
[07/50] [abbrv] incubator-usergrid git commit: Migrations work. Need
to finish tests.
Posted by sf...@apache.org.
Migrations work. Need to finish tests.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/a55c784d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/a55c784d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/a55c784d
Branch: refs/heads/USERGRID-480
Commit: a55c784dac5beb624950f6b09746667e4431fbd8
Parents: e96af46
Author: Todd Nine <tn...@apigee.com>
Authored: Mon Mar 2 16:11:11 2015 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Mon Mar 2 17:19:27 2015 -0700
----------------------------------------------------------------------
.../collection/guice/CollectionModule.java | 12 +-
.../impl/CollectionDataVersions.java | 4 +-
.../MvccEntitySerializationStrategyV1Impl.java | 2 +-
.../MvccEntitySerializationStrategyV2Impl.java | 2 +-
.../MvccEntitySerializationStrategyV3Impl.java | 2 +-
.../serialization/impl/SerializationModule.java | 78 ++--
.../migration/CollectionMigrationPlugin.java | 78 +++-
.../migration/MvccEntityDataMigrationImpl.java | 446 ++++++++++++++-----
.../collection/guice/TestCollectionModule.java | 17 +-
.../persistence/core/guice/CommonModule.java | 2 +
.../data/MigrationInfoSerializationImpl.java | 6 +-
.../migration/data/newimpls/DataMigration2.java | 17 +-
.../data/newimpls/MigrationPlugin.java | 6 +-
.../newimpls/TestMigrationDataProvider.java | 61 +++
14 files changed, 563 insertions(+), 170 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a55c784d/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/guice/CollectionModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/guice/CollectionModule.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/guice/CollectionModule.java
index af5c4cf..7d78177 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/guice/CollectionModule.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/guice/CollectionModule.java
@@ -55,7 +55,7 @@ import com.google.inject.multibindings.Multibinder;
*
* @author tnine
*/
-public class CollectionModule extends AbstractModule {
+public abstract class CollectionModule extends AbstractModule {
@Override
@@ -86,6 +86,8 @@ public class CollectionModule extends AbstractModule {
bind( UniqueValueSerializationStrategy.class ).to( UniqueValueSerializationStrategyImpl.class );
bind( ChangeLogGenerator.class).to( ChangeLogGeneratorImpl.class);
+ configureMigrationProvider();
+
}
@Provides
@@ -120,6 +122,14 @@ public class CollectionModule extends AbstractModule {
}
+ /**
+ * Gives callers the ability to to configure an instance of
+ *
+ * MigrationDataProvider<EntityIdScope> for providing data migrations
+ */
+ public abstract void configureMigrationProvider();
+
+
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a55c784d/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/CollectionDataVersions.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/CollectionDataVersions.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/CollectionDataVersions.java
index ee84b1e..6e8be45 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/CollectionDataVersions.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/CollectionDataVersions.java
@@ -28,9 +28,9 @@ package org.apache.usergrid.persistence.collection.serialization.impl;
* Versions of data as they exist across our system
*/
public enum CollectionDataVersions{
+ ZERO(0),
ONE(1),
- TWO(2),
- THREE(3);
+ TWO(2);
private final int version;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a55c784d/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV1Impl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV1Impl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV1Impl.java
index a959508..1dab673 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV1Impl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV1Impl.java
@@ -90,7 +90,7 @@ public class MvccEntitySerializationStrategyV1Impl extends MvccEntitySerializati
@Override
public int getImplementationVersion() {
- return CollectionDataVersions.ONE.getVersion();
+ return CollectionDataVersions.ZERO.getVersion();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a55c784d/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2Impl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2Impl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2Impl.java
index 04a1fea..1f65fcb 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2Impl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2Impl.java
@@ -93,7 +93,7 @@ public class MvccEntitySerializationStrategyV2Impl extends MvccEntitySerializati
@Override
public int getImplementationVersion() {
- return CollectionDataVersions.TWO.getVersion();
+ return CollectionDataVersions.ONE.getVersion();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a55c784d/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3Impl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3Impl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3Impl.java
index 4e73119..7b8aac1 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3Impl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3Impl.java
@@ -383,7 +383,7 @@ public class MvccEntitySerializationStrategyV3Impl implements MvccEntitySerializ
@Override
public int getImplementationVersion() {
- return CollectionDataVersions.THREE.getVersion();
+ return CollectionDataVersions.TWO.getVersion();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a55c784d/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/SerializationModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/SerializationModule.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/SerializationModule.java
index fe804f3..f32f4f9 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/SerializationModule.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/SerializationModule.java
@@ -34,7 +34,7 @@ import org.apache.usergrid.persistence.core.migration.schema.Migration;
import com.google.inject.AbstractModule;
import com.google.inject.Inject;
import com.google.inject.Key;
-import com.google.inject.Provides;
+import com.google.inject.Provider;
import com.google.inject.Singleton;
import com.google.inject.TypeLiteral;
import com.google.inject.multibindings.Multibinder;
@@ -54,26 +54,32 @@ public class SerializationModule extends AbstractModule {
//We've migrated this one, so we need to set up the previous, current, and proxy
- bind(MvccEntitySerializationStrategy.class).annotatedWith( ProxyImpl.class ).to( MvccEntitySerializationStrategyProxyImpl.class );
-
-
+ bind( MvccEntitySerializationStrategy.class ).annotatedWith( ProxyImpl.class )
+ .to( MvccEntitySerializationStrategyProxyImpl.class );
//bind all 3 implementations
- bind(MvccEntitySerializationStrategyV1Impl.class);
- bind(MvccEntitySerializationStrategyV2Impl.class);
- bind(MvccEntitySerializationStrategyV3Impl.class);
+ bind( MvccEntitySerializationStrategyV1Impl.class );
+ bind( MvccEntitySerializationStrategyV2Impl.class );
+ bind( MvccEntitySerializationStrategyV3Impl.class );
+
+ bind( new TypeLiteral<VersionedMigrationSet<MvccEntitySerializationStrategy>>() {} )
+ .toProvider( MvccEntitySerializationStrategyProvider.class );
//migrations
//we want to make sure our generics are retained, so we use a typeliteral
- Multibinder<DataMigration2<EntityIdScope>> dataMigrationMultibinder = Multibinder.newSetBinder( binder(), new TypeLiteral<DataMigration2<EntityIdScope>>(){});
+ Multibinder<DataMigration2<EntityIdScope>> dataMigrationMultibinder =
+ Multibinder.newSetBinder( binder(), new TypeLiteral<DataMigration2<EntityIdScope>>() {} );
+
+
dataMigrationMultibinder.addBinding().to( MvccEntityDataMigrationImpl.class );
//wire up the collection migration plugin
- Multibinder.newSetBinder( binder(), MigrationPlugin.class).addBinding().to( CollectionMigrationPlugin.class );
+ Multibinder.newSetBinder( binder(), MigrationPlugin.class ).addBinding().to( CollectionMigrationPlugin.class );
+
@@ -90,39 +96,53 @@ public class SerializationModule extends AbstractModule {
//bind our settings as an eager singleton so it's checked on startup
- bind(SettingsValidation.class).asEagerSingleton();
+ bind( SettingsValidation.class ).asEagerSingleton();
}
- /**
- * Configure via explicit declaration the migration path we can follow
- * @param v1
- * @param v2
- * @param v3
- * @return
- */
@Singleton
- @Inject
- @Provides
- public VersionedMigrationSet<MvccEntitySerializationStrategy> getVersions(final MvccEntitySerializationStrategyV1Impl v1, final MvccEntitySerializationStrategyV2Impl v2, final MvccEntitySerializationStrategyV3Impl v3){
+ public static final class MvccEntitySerializationStrategyProvider
+ implements Provider<VersionedMigrationSet<MvccEntitySerializationStrategy>> {
+
+
+ private final MvccEntitySerializationStrategyV1Impl v1;
+ private final MvccEntitySerializationStrategyV2Impl v2;
+ private final MvccEntitySerializationStrategyV3Impl v3;
+
+
+ @Inject
+ public MvccEntitySerializationStrategyProvider( final MvccEntitySerializationStrategyV1Impl v1,
+ final MvccEntitySerializationStrategyV2Impl v2,
+ final MvccEntitySerializationStrategyV3Impl v3 ) {
+ this.v1 = v1;
+ this.v2 = v2;
+ this.v3 = v3;
+ }
+
+ @Override
+ public VersionedMigrationSet<MvccEntitySerializationStrategy> get() {
- //we must perform a migration from v1 to v3 in order to maintain consistency
- MigrationRelationship<MvccEntitySerializationStrategy> v1Tov3 = new MigrationRelationship<>( v1, v3 );
+ //we must perform a migration from v1 to v3 in order to maintain consistency
+ MigrationRelationship<MvccEntitySerializationStrategy> v1Tov3 = new MigrationRelationship<>( v1, v3 );
- //we must migrate from 2 to 3, this is a bridge that must happen to maintain data consistency
+ //we must migrate from 2 to 3, this is a bridge that must happen to maintain data consistency
- MigrationRelationship<MvccEntitySerializationStrategy> v2Tov3 = new MigrationRelationship<>( v2, v3 );
+ MigrationRelationship<MvccEntitySerializationStrategy> v2Tov3 = new MigrationRelationship<>( v2, v3 );
- //note that we MUST migrate to v3 before our next migration, if v4 and v5 is implemented we will need a v3->v5 and a v4->v5 set
- MigrationRelationship<MvccEntitySerializationStrategy> current = new MigrationRelationship<MvccEntitySerializationStrategy>( v3, v3 );
+ //note that we MUST migrate to v3 before our next migration, if v4 and v5 is implemented we will need a
+ // v3->v5 and a v4->v5 set
+ MigrationRelationship<MvccEntitySerializationStrategy> current =
+ new MigrationRelationship<MvccEntitySerializationStrategy>( v3, v3 );
- //now create our set of versions
- VersionedMigrationSet<MvccEntitySerializationStrategy> set = new VersionedMigrationSet<>( v1Tov3, v2Tov3, current );
+ //now create our set of versions
+ VersionedMigrationSet<MvccEntitySerializationStrategy> set =
+ new VersionedMigrationSet<>( v1Tov3, v2Tov3, current );
- return set;
+ return set;
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a55c784d/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/CollectionMigrationPlugin.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/CollectionMigrationPlugin.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/CollectionMigrationPlugin.java
index 5d52a9f..d0663c2 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/CollectionMigrationPlugin.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/CollectionMigrationPlugin.java
@@ -26,6 +26,11 @@ package org.apache.usergrid.persistence.collection.serialization.impl.migration;
import java.util.Set;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.usergrid.persistence.core.migration.data.DataMigrationException;
+import org.apache.usergrid.persistence.core.migration.data.MigrationInfoSerialization;
import org.apache.usergrid.persistence.core.migration.data.newimpls.DataMigration2;
import org.apache.usergrid.persistence.core.migration.data.newimpls.MigrationDataProvider;
import org.apache.usergrid.persistence.core.migration.data.newimpls.MigrationPlugin;
@@ -42,17 +47,22 @@ import com.google.inject.Singleton;
public class CollectionMigrationPlugin implements MigrationPlugin {
- public static final String PLUGIN_NAME = "collections-entity-data";
+ private static final Logger LOG = LoggerFactory.getLogger( CollectionMigrationPlugin.class );
+
+ public static final String PLUGIN_NAME = "collections-entity-data";
- private final DataMigration2<EntityIdScope> entityDataMigration;
+ private final Set<DataMigration2<EntityIdScope>> entityDataMigrations;
private final MigrationDataProvider<EntityIdScope> entityIdScopeDataMigrationProvider;
+ private final MigrationInfoSerialization migrationInfoSerialization;
@Inject
- public CollectionMigrationPlugin( final DataMigration2<EntityIdScope> entityDataMigration,
- final MigrationDataProvider<EntityIdScope> entityIdScopeDataMigrationProvider ) {
- this.entityDataMigration = entityDataMigration;
+ public CollectionMigrationPlugin( final Set<DataMigration2<EntityIdScope>> entityDataMigrations,
+ final MigrationDataProvider<EntityIdScope> entityIdScopeDataMigrationProvider,
+ final MigrationInfoSerialization migrationInfoSerialization ) {
+ this.entityDataMigrations = entityDataMigrations;
this.entityIdScopeDataMigrationProvider = entityIdScopeDataMigrationProvider;
+ this.migrationInfoSerialization = migrationInfoSerialization;
}
@@ -64,12 +74,66 @@ public class CollectionMigrationPlugin implements MigrationPlugin {
@Override
public void run( final ProgressObserver observer ) {
- entityDataMigration.migrate( entityIdScopeDataMigrationProvider, observer );
+
+ //run until complete
+ while(runMigration( observer )){
+ LOG.info( "Migration complete, checking for next run" );
+ }
+
}
@Override
public int getMaxVersion() {
- return 0;
+
+ int max = 0;
+
+ for(DataMigration2<EntityIdScope> entityMigration: entityDataMigrations){
+ max = Math.max( max, entityMigration.getMaxVersion() );
+ }
+
+ return max;
+ }
+
+
+ /**
+ * Try to run the migration
+ *
+ * @return True if we ran a migration
+ */
+ private boolean runMigration( final ProgressObserver po ) {
+ DataMigration2<EntityIdScope> migrationToExecute = null;
+
+
+ final int version = migrationInfoSerialization.getVersion( PLUGIN_NAME );
+
+ for ( DataMigration2<EntityIdScope> entityMigration : entityDataMigrations ) {
+ if ( entityMigration.supports( version ) ) {
+ if ( migrationToExecute != null ) {
+ throw new DataMigrationException(
+ "Two migrations attempted to migration the same version, this is not allowed. Class '"
+ + migrationToExecute.getClass().getName() + "' and class '" + entityMigration
+ .getClass().getName()
+ + "' both support this version. This means something is wired incorrectly" );
+ }
+
+ migrationToExecute = entityMigration;
+ }
+ }
+
+ if(migrationToExecute == null){
+ LOG.info( "No migrations found to execute" );
+ return false;
+ }
+
+ //run the migration
+ final int newSystemVersion = migrationToExecute.migrate( version, entityIdScopeDataMigrationProvider, po );
+
+ migrationInfoSerialization.setVersion( PLUGIN_NAME, newSystemVersion );
+
+ //signal we've run a migration and return
+ return true;
+
+
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a55c784d/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/MvccEntityDataMigrationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/MvccEntityDataMigrationImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/MvccEntityDataMigrationImpl.java
index 5250b5c..48e0195 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/MvccEntityDataMigrationImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/MvccEntityDataMigrationImpl.java
@@ -26,10 +26,14 @@ import java.util.UUID;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.usergrid.persistence.collection.CollectionScope;
+import org.apache.usergrid.persistence.collection.EntityVersionCleanupFactory;
import org.apache.usergrid.persistence.collection.MvccEntity;
+import org.apache.usergrid.persistence.collection.impl.EntityVersionCleanupTask;
import org.apache.usergrid.persistence.collection.serialization.MvccEntitySerializationStrategy;
import org.apache.usergrid.persistence.collection.serialization.UniqueValue;
import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
+import org.apache.usergrid.persistence.collection.serialization.impl.MvccEntitySerializationStrategyImpl;
+import org.apache.usergrid.persistence.collection.serialization.impl.MvccEntitySerializationStrategyV3Impl;
import org.apache.usergrid.persistence.collection.serialization.impl.UniqueValueImpl;
import org.apache.usergrid.persistence.collection.util.EntityUtils;
import org.apache.usergrid.persistence.core.migration.data.DataMigrationException;
@@ -45,51 +49,68 @@ import org.apache.usergrid.persistence.model.field.Field;
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
import com.google.inject.Inject;
+import com.google.inject.Singleton;
import com.netflix.astyanax.Keyspace;
import com.netflix.astyanax.MutationBatch;
import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
import rx.Observable;
import rx.Subscriber;
+import rx.functions.Action0;
import rx.functions.Action1;
import rx.functions.Func1;
import rx.functions.Func2;
+import rx.observables.GroupedObservable;
import rx.schedulers.Schedulers;
/**
* Data migration strategy for entities
*/
+@Singleton
public class MvccEntityDataMigrationImpl implements DataMigration2<EntityIdScope> {
private final Keyspace keyspace;
private final VersionedMigrationSet<MvccEntitySerializationStrategy> allVersions;
private final UniqueValueSerializationStrategy uniqueValueSerializationStrategy;
private final MigrationInfoSerialization migrationInfoSerialization;
+ private final EntityVersionCleanupFactory entityVersionCleanupFactory;
+ private final MvccEntitySerializationStrategyV3Impl mvccEntitySerializationStrategyV3;
@Inject
public MvccEntityDataMigrationImpl( final Keyspace keyspace,
final VersionedMigrationSet<MvccEntitySerializationStrategy> allVersions,
final UniqueValueSerializationStrategy uniqueValueSerializationStrategy,
- final MigrationInfoSerialization migrationInfoSerialization ) {
+ final MigrationInfoSerialization migrationInfoSerialization,
+ final EntityVersionCleanupFactory entityVersionCleanupFactory,
+ final MvccEntitySerializationStrategyV3Impl mvccEntitySerializationStrategyV3 ) {
this.keyspace = keyspace;
this.allVersions = allVersions;
this.uniqueValueSerializationStrategy = uniqueValueSerializationStrategy;
this.migrationInfoSerialization = migrationInfoSerialization;
+ this.entityVersionCleanupFactory = entityVersionCleanupFactory;
+ this.mvccEntitySerializationStrategyV3 = mvccEntitySerializationStrategyV3;
}
@Override
- public int getVersion() {
- //get the max implementation version, since that's what we're going to
- return allVersions.getMaxVersion( migrationInfoSerialization.getVersion( CollectionMigrationPlugin.PLUGIN_NAME ) );
+ public boolean supports( final int currentVersion ) {
+ //we can only migrate up to v3 with this implementation. Beyond that, we should use a different migration
+ return currentVersion < mvccEntitySerializationStrategyV3.getImplementationVersion();
}
@Override
- public void migrate( final MigrationDataProvider<EntityIdScope> migrationDataProvider, final ProgressObserver observer ) {
+ public int getMaxVersion() {
+ return mvccEntitySerializationStrategyV3.getImplementationVersion();
+ }
+
+
+ @Override
+ public int migrate( final int currentVersion, final MigrationDataProvider<EntityIdScope> migrationDataProvider,
+ final ProgressObserver observer ) {
final AtomicLong atomicLong = new AtomicLong();
@@ -97,137 +118,326 @@ public class MvccEntityDataMigrationImpl implements DataMigration2<EntityIdScope
final UUID startTime = UUIDGenerator.newTimeUUID();
- final MigrationRelationship<MvccEntitySerializationStrategy>
- migration = allVersions.getMigrationRelationship( getCurrentSystemVersion() );
-
- final long migrated = migrationDataProvider.getData().subscribeOn( Schedulers.io() )
- .parallel( new Func1<Observable<EntityIdScope>, Observable<Long>>() {
-
-
- //process the ids in parallel
- @Override
- public Observable<Long> call( final Observable<EntityIdScope> entityIdScopeObservable ) {
-
-
- return entityIdScopeObservable
- .flatMap( new Func1<EntityIdScope, Observable<EntityToSaveMessage>>() {
-
-
- @Override
- public Observable<EntityToSaveMessage> call(
- final EntityIdScope entityIdScope ) {
-
- //load the entity
- final CollectionScope currentScope = entityIdScope.getCollectionScope();
-
-
-
- //for each element in our history, we need to copy it to v2. Note that this migration
- //won't support anything beyond V2
-
- final Iterator<MvccEntity> allVersions = migration.from
- .loadAscendingHistory( currentScope, entityIdScope.getId(),
- startTime, 100 );
-
- //emit all the entities
- return Observable.create( new Observable.OnSubscribe<EntityToSaveMessage>() {
- @Override
- public void call(
- final Subscriber<? super EntityToSaveMessage> subscriber ) {
-
- while ( allVersions.hasNext() ) {
- final EntityToSaveMessage message =
- new EntityToSaveMessage( currentScope,
- allVersions.next() );
- subscriber.onNext( message );
- }
-
- subscriber.onCompleted();
- }
- } );
- }
- } ).buffer( 100 ).doOnNext( new Action1<List<EntityToSaveMessage>>() {
- @Override
- public void call( final List<EntityToSaveMessage> messages ) {
- atomicLong.addAndGet( messages.size() );
-
- final MutationBatch totalBatch = keyspace.prepareMutationBatch();
+ final MigrationRelationship<MvccEntitySerializationStrategy> migration =
+ allVersions.getMigrationRelationship( currentVersion );
- for ( EntityToSaveMessage message : messages ) {
-
- final MutationBatch entityRewrite =
- migration.to.write( message.scope, message.entity );
-
- //add to the total batch
- totalBatch.mergeShallow( entityRewrite );
-
- //write the unique values
-
- if ( message.entity.getEntity().isPresent() ) {
-
- final Entity entity = message.entity.getEntity().get();
-
- final Id entityId = entity.getId();
-
- final UUID version = message.entity.getVersion();
-
- // re-write the unique values but this time with no TTL
- for ( Field field : EntityUtils
- .getUniqueFields( message.entity.getEntity().get() ) ) {
-
- UniqueValue written =
- new UniqueValueImpl( field, entityId, version );
-
- MutationBatch mb =
- uniqueValueSerializationStrategy.write( message.scope, written );
-
-
- // merge into our existing mutation batch
- totalBatch.mergeShallow( mb );
- }
-
-
- }
- }
-
-
- executeBatch( totalBatch, observer, atomicLong );
- }
- } )
- //count the results
- .reduce( 0l, new Func2<Long, List<EntityToSaveMessage>, Long>() {
- @Override
- public Long call( final Long aLong, final List<EntityToSaveMessage> ids ) {
- return aLong + ids.size();
- }
- } );
- }
- } ).toBlocking().last();
+ final long migrated = migrationDataProvider.getData().subscribeOn( Schedulers.io() )
+ .parallel( new Func1<Observable<EntityIdScope>, Observable<Long>>() {
+
+
+ //process the ids in parallel
+ @Override
+ public Observable<Long> call(
+ final Observable<EntityIdScope>
+ entityIdScopeObservable ) {
+
+
+ return entityIdScopeObservable.flatMap(
+ new Func1<EntityIdScope,
+ Observable<EntityToSaveMessage>>() {
+
+
+ @Override
+ public
+ Observable<EntityToSaveMessage> call(
+ final EntityIdScope
+ entityIdScope ) {
+
+ //load the entity
+ final CollectionScope
+ currentScope =
+ entityIdScope
+ .getCollectionScope();
+
+
+ //for each element in our
+ // history, we need to copy it
+ // to v2.
+ // Note that
+ // this migration
+ //won't support anything beyond V2
+
+ final Iterator<MvccEntity>
+ allVersions =
+ migration.from
+ .loadAscendingHistory(
+ currentScope,
+ entityIdScope
+ .getId(),
+ startTime,
+ 100 );
+
+ //emit all the entity versions
+ return Observable.create(
+ new Observable
+ .OnSubscribe<EntityToSaveMessage>() {
+ @Override
+ public void call(
+ final
+ Subscriber<? super
+ EntityToSaveMessage> subscriber ) {
+
+ while ( allVersions
+ .hasNext() ) {
+ final
+ EntityToSaveMessage
+ message =
+ new EntityToSaveMessage(
+ currentScope,
+ allVersions
+ .next() );
+ subscriber.onNext( message );
+ }
+
+ subscriber.onCompleted();
+ }
+ } );
+ }
+ } )
+
+
+ //group them by entity id so we can get
+ // the max for cleanup
+ .groupBy(
+ new Func1<EntityToSaveMessage,
+ Id>() {
+ @Override
+ public Id call(
+ final
+ EntityToSaveMessage
+ entityToSaveMessage ) {
+ return entityToSaveMessage.entity
+ .getId();
+ }
+ } )
+ //buffer up 10 of groups so we can put them all in a single mutation
+ .buffer( 10 ).doOnNext(
+ new Action1<List<GroupedObservable<Id, EntityToSaveMessage>>>() {
+
+
+ @Override
+ public void call(
+ final
+ List<GroupedObservable<Id, EntityToSaveMessage>> groupedObservables ) {
+
+ atomicLong.addAndGet(
+ groupedObservables
+ .size() );
+
+ final MutationBatch
+ totalBatch =
+ keyspace.prepareMutationBatch();
+
+
+ //run each of the
+ // groups and add
+ // it ot the batch
+ Observable
+ .from( groupedObservables )
+ //emit the group as an observable
+ .flatMap(
+ new Func1<GroupedObservable<Id, EntityToSaveMessage>, Observable<EntityToSaveMessage>>() {
+
+
+ @Override
+ public Observable<EntityToSaveMessage> call(
+ final GroupedObservable<Id, EntityToSaveMessage> idEntityToSaveMessageGroupedObservable ) {
+ return idEntityToSaveMessageGroupedObservable
+ .asObservable();
+ }
+ } )
+
+ //merge and add the batch
+ .doOnNext(
+ new Action1<EntityToSaveMessage>() {
+ @Override
+ public void call(
+ final EntityToSaveMessage message ) {
+
+ final MutationBatch
+ entityRewrite =
+ migration.to.write( message.scope,
+ message.entity );
+
+ //add to
+ // the
+ // total
+ // batch
+ totalBatch.mergeShallow( entityRewrite );
+
+ //write
+ // the
+ // unique values
+
+ if ( !message.entity
+ .getEntity()
+ .isPresent() ) {
+ return;
+ }
+
+ final Entity
+ entity =
+ message.entity
+ .getEntity()
+ .get();
+
+ final Id
+ entityId =
+ entity.getId();
+
+ final UUID
+ version =
+ message.entity
+ .getVersion();
+
+ // re-write the unique values
+ // but this
+ // time with
+ // no TTL so that cleanup can clean up older values
+ for ( Field field : EntityUtils
+ .getUniqueFields(
+ message.entity
+ .getEntity()
+ .get() ) ) {
+
+ UniqueValue
+ written =
+ new UniqueValueImpl(
+ field,
+ entityId,
+ version );
+
+ MutationBatch
+ mb =
+ uniqueValueSerializationStrategy
+ .write( message.scope,
+ written );
+
+
+ // merge into our
+ // existing mutation
+ // batch
+ totalBatch
+ .mergeShallow(
+ mb );
+ }
+ }
+ } )
+ //once we've streamed everything, flush it
+ .doOnCompleted(
+
+ new Action0() {
+ @Override
+ public void call() {
+
+ executeBatch(migration.to.getImplementationVersion(), totalBatch, observer, atomicLong );
+ }
+ } )
+ .toBlocking()
+ .last();
+ }
+ } ).doOnNext(
+ new Action1<List<GroupedObservable<Id, EntityToSaveMessage>>>() {
+ @Override
+ public void call(
+ final List<GroupedObservable<Id, EntityToSaveMessage>> groupedObservables ) {
+
+ for ( final GroupedObservable<Id, EntityToSaveMessage> group : groupedObservables ) {
+
+ //get the highest
+ // entity and run a
+ // cleanup task on it
+ final EntityToSaveMessage
+ maxEntity =
+ group.toBlocking()
+ .last();
+
+ final EntityVersionCleanupTask
+ task =
+ entityVersionCleanupFactory
+ .getTask(
+ maxEntity.scope,
+ maxEntity.entity
+ .getId(),
+ maxEntity.entity
+ .getVersion() );
+
+ /**
+ * just run the
+ * call in this
+ * process, we're
+ * already
+ * doing parallel
+ * this forces a
+ * repair of the
+ * unique properties,
+ and will bring us
+ to a consistent
+ state after the
+ */
+
+ try {
+ task.call();
+ }
+ catch ( Exception e ) {
+ throw new RuntimeException(
+ "Unable to run cleanup task",
+ e );
+ }
+ }
+ }
+ } ).
+ reduce( 0l,
+ new Func2<Long, List<GroupedObservable<Id, EntityToSaveMessage>>, Long>() {
+
+ @Override
+ public Long call(
+ final Long aLong,
+ final List<GroupedObservable<Id, EntityToSaveMessage>> groupedObservables ) {
+
+ long newCount =
+ aLong;
+
+ for ( GroupedObservable<Id, EntityToSaveMessage> group : groupedObservables ) {
+ newCount +=
+ group.longCount()
+ .toBlocking()
+ .last();
+ }
+
+ return newCount;
+ }
+ }
+
+
+ );
+ }}).toBlocking().last();
//now we update the progress observer
- observer.update( getVersion(), "Finished for this step. Migrated " + migrated + "entities total. ");
+ observer.update( migration.to.getImplementationVersion(), "Finished for this step. Migrated " + migrated + "entities total. " );
+
+ return migration.to.getImplementationVersion();
}
- protected void executeBatch( final MutationBatch batch, final ProgressObserver po, final AtomicLong count ) {
+ protected void executeBatch( final int targetVersion, final MutationBatch batch, final ProgressObserver po, final AtomicLong count ) {
try {
batch.execute();
- po.update( getVersion(), "Finished copying " + count + " entities to the new format" );
+ po.update(targetVersion, "Finished copying " + count + " entities to the new format" );
}
catch ( ConnectionException e ) {
- po.failed( getVersion(), "Failed to execute mutation in cassandra" );
+ po.failed( targetVersion, "Failed to execute mutation in cassandra" );
throw new DataMigrationException( "Unable to migrate batches ", e );
}
}
- private int getCurrentSystemVersion(){
- return migrationInfoSerialization.getVersion( CollectionMigrationPlugin.PLUGIN_NAME );
- }
- private static final class EntityToSaveMessage{
+
+
+ private static final class EntityToSaveMessage {
private final CollectionScope scope;
private final MvccEntity entity;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a55c784d/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/guice/TestCollectionModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/guice/TestCollectionModule.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/guice/TestCollectionModule.java
index 7e1f32b..b42ad94 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/guice/TestCollectionModule.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/guice/TestCollectionModule.java
@@ -20,8 +20,15 @@
package org.apache.usergrid.persistence.collection.guice;
+import java.util.Collections;
+
+import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
import org.apache.usergrid.persistence.core.guice.CommonModule;
import org.apache.usergrid.persistence.core.guice.TestModule;
+import org.apache.usergrid.persistence.core.migration.data.newimpls.MigrationDataProvider;
+import org.apache.usergrid.persistence.core.migration.data.newimpls.TestMigrationDataProvider;
+
+import com.google.inject.TypeLiteral;
public class TestCollectionModule extends TestModule {
@@ -30,7 +37,15 @@ public class TestCollectionModule extends TestModule {
protected void configure() {
install( new CommonModule() );
- install( new CollectionModule() );
+ install( new CollectionModule() {
+ @Override
+ public void configureMigrationProvider() {
+ //configure our migration data provider
+
+ TestMigrationDataProvider<EntityIdScope> migrationDataProvider = new TestMigrationDataProvider<>();
+ bind(new TypeLiteral< MigrationDataProvider<EntityIdScope>>(){}).toInstance( migrationDataProvider );
+ }
+ } );
/**
* Test modules
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a55c784d/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/guice/CommonModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/guice/CommonModule.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/guice/CommonModule.java
index 61b6b04..97c0479 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/guice/CommonModule.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/guice/CommonModule.java
@@ -76,6 +76,8 @@ public class CommonModule extends AbstractModule {
bind( DataMigrationManager.class ).to( DataMigrationManagerImpl.class );
+ bind (MigrationInfoCache.class).to( MigrationInfoCacheImpl.class );
+
//do multibindings for migrations
//create the empty multibinder so other plugins can use it
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a55c784d/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/MigrationInfoSerializationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/MigrationInfoSerializationImpl.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/MigrationInfoSerializationImpl.java
index f976568..4a349fd 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/MigrationInfoSerializationImpl.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/MigrationInfoSerializationImpl.java
@@ -146,7 +146,7 @@ public class MigrationInfoSerializationImpl implements MigrationInfoSerializatio
}
//swallow, it doesn't exist
catch ( NotFoundException nfe ) {
- return -1;
+ return 0;
}
catch ( ConnectionException e ) {
throw new DataMigrationException( "Unable to retrieve status", e );
@@ -180,7 +180,7 @@ public class MigrationInfoSerializationImpl implements MigrationInfoSerializatio
}
//swallow, it doesn't exist
catch ( NotFoundException nfe ) {
- return -1;
+ return 0;
}
catch ( ConnectionException e ) {
throw new DataMigrationException( "Unable to retrieve status", e );
@@ -196,7 +196,7 @@ public class MigrationInfoSerializationImpl implements MigrationInfoSerializatio
}
//swallow, it doesn't exist
catch ( NotFoundException nfe ) {
- return -1;
+ return 0;
}
catch ( ConnectionException e ) {
throw new DataMigrationException( "Unable to retrieve status", e );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a55c784d/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/DataMigration2.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/DataMigration2.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/DataMigration2.java
index 5830335..303c11c 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/DataMigration2.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/DataMigration2.java
@@ -33,15 +33,26 @@ public interface DataMigration2<T> {
/**
* Perform the migration, returning an observable with a single emitted value
+ * @param currentVersion the current version of the system
* @param migrationDataProvider
+ * @param observer The observer to receive updates of the progress
+ *
+ * @return The version that the system is now running
*/
- public void migrate(MigrationDataProvider<T> migrationDataProvider, ProgressObserver observer);
+ public int migrate(final int currentVersion, MigrationDataProvider<T> migrationDataProvider, ProgressObserver observer);
/**
- * Get the version of this migration. It should be unique within the scope of the plugin
+ * Check if this version supports migration from the current system version. If this returns false,
+ * migrate will not be invoked
* @return
*/
- public int getVersion();
+ public boolean supports(final int currentVersion);
+
+ /**
+ * Get the max version this migration can migrate to
+ * @return
+ */
+ public int getMaxVersion();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a55c784d/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/MigrationPlugin.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/MigrationPlugin.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/MigrationPlugin.java
index 880cfd1..50dc91b 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/MigrationPlugin.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/MigrationPlugin.java
@@ -42,10 +42,10 @@ public interface MigrationPlugin {
*/
public void run(ProgressObserver observer);
+
/**
- * Get the maximum migration version this plugin implements
+ * Get the max version this plugin supports
* @return
*/
- public int getMaxVersion();
-
+ int getMaxVersion();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a55c784d/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/newimpls/TestMigrationDataProvider.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/newimpls/TestMigrationDataProvider.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/newimpls/TestMigrationDataProvider.java
new file mode 100644
index 0000000..9e99c6f
--- /dev/null
+++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/newimpls/TestMigrationDataProvider.java
@@ -0,0 +1,61 @@
+/*
+ *
+ * *
+ * * Licensed to the Apache Software Foundation (ASF) under one
+ * * or more contributor license agreements. See the NOTICE file
+ * * distributed with this work for additional information
+ * * regarding copyright ownership. The ASF licenses this file
+ * * to you under the Apache License, Version 2.0 (the
+ * * "License"); you may not use this file except in compliance
+ * * with the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing,
+ * * software distributed under the License is distributed on an
+ * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * * KIND, either express or implied. See the License for the
+ * * specific language governing permissions and limitations
+ * * under the License.
+ * *
+ *
+ */
+
+package org.apache.usergrid.persistence.core.migration.data.newimpls;
+
+
+import java.util.Collection;
+
+import rx.Observable;
+
+
+/**
+ * A simple test class that will emit the provided test data when subscribed
+ * @param <T>
+ */
+public class TestMigrationDataProvider<T> implements MigrationDataProvider<T> {
+
+
+
+ //default to nothing so that we don't return null
+ private Observable<T> observable = Observable.empty();
+
+
+ public TestMigrationDataProvider( ) {}
+
+
+ @Override
+ public Observable<T> getData() {
+ return observable;
+ }
+
+
+ /**
+ * Set this observable to return when invoked
+ *
+ * @param observable
+ */
+ public void setObservable( final Observable<T> observable ) {
+ this.observable = observable;
+ }
+}
[36/50] [abbrv] incubator-usergrid git commit: Merge branch
'two-dot-o' into USERGRID-405
Posted by sf...@apache.org.
Merge branch 'two-dot-o' into USERGRID-405
Conflicts:
stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java
stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityCollectionManager.java
stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/guice/CollectionModule.java
stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerFactoryImpl.java
stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityDeletedTask.java
stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTask.java
stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/UniqueValueSerializationStrategy.java
stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/util/EntityUtils.java
stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTaskTest.java
stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java
stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/MapManager.java
stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/7b5d2224
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/7b5d2224
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/7b5d2224
Branch: refs/heads/USERGRID-480
Commit: 7b5d2224512f0043336acbd9b9323e1e32edf888
Parents: 5c7a5f8 b53cb07
Author: Todd Nine <tn...@apigee.com>
Authored: Tue Mar 17 11:36:08 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Tue Mar 17 11:36:08 2015 -0600
----------------------------------------------------------------------
.../main/dist/init_instance/init_rest_server.sh | 2 +
.../dist/init_instance/install_elasticsearch.sh | 22 +-
.../main/groovy/configure_elasticsearch.groovy | 80 ++--
.../src/main/groovy/configure_usergrid.groovy | 12 +-
stack/awscluster/ugcluster-cf.json | 466 +++++++++++++------
stack/core/pom.xml | 10 -
.../batch/service/JobSchedulerService.java | 5 -
.../usergrid/corepersistence/CoreModule.java | 36 +-
.../corepersistence/CpEntityManager.java | 64 ++-
.../corepersistence/CpEntityManagerFactory.java | 4 -
.../corepersistence/CpRelationManager.java | 14 -
.../events/EntityDeletedHandler.java | 58 ++-
.../events/EntityVersionCreatedHandler.java | 60 ++-
.../events/EntityVersionDeletedHandler.java | 53 ++-
.../results/FilteringLoader.java | 2 +-
.../usergrid/persistence/EntityManager.java | 10 +-
.../persistence/EntityManagerFactory.java | 2 -
.../cassandra/EntityManagerFactoryImpl.java | 3 -
.../cassandra/EntityManagerImpl.java | 31 +-
.../cassandra/RelationManagerImpl.java | 58 +--
.../cassandra/index/ConnectedIndexScanner.java | 2 -
.../cassandra/index/IndexBucketScanner.java | 2 -
.../corepersistence/StaleIndexCleanupTest.java | 38 +-
.../usergrid/persistence/EntityManagerIT.java | 1 +
.../PerformanceEntityRebuildIndexTest.java | 2 +
.../collection/EntityCollectionManager.java | 20 +-
.../collection/EntityDeletedFactory.java | 34 --
.../collection/EntityVersionCleanupFactory.java | 35 --
.../collection/EntityVersionCreatedFactory.java | 31 --
.../persistence/collection/FieldSet.java | 48 ++
.../cache/CachedEntityCollectionManager.java | 11 +-
.../collection/guice/CollectionModule.java | 11 +-
.../EntityCollectionManagerFactoryImpl.java | 32 +-
.../impl/EntityCollectionManagerImpl.java | 251 ++++++++--
.../collection/impl/EntityDeletedTask.java | 9 +-
.../impl/EntityVersionCleanupTask.java | 34 +-
.../impl/EntityVersionTaskFactory.java | 65 +++
.../mvcc/stage/write/WriteUniqueVerify.java | 140 +++---
.../UniqueValueSerializationStrategy.java | 12 +
.../serialization/impl/MutableFieldSet.java | 63 +++
.../UniqueValueSerializationStrategyImpl.java | 27 +-
.../collection/util/EntityUtils.java | 4 +-
.../collection/EntityCollectionManagerIT.java | 68 +++
.../mvcc/stage/write/WriteUniqueVerifyTest.java | 6 +-
.../core/astyanax/CassandraConfig.java | 6 +
.../core/astyanax/CassandraConfigImpl.java | 8 +-
.../persistence/core/astyanax/CassandraFig.java | 9 +-
.../persistence/core/future/BetterFuture.java | 43 +-
.../core/metrics/MetricsFactory.java | 9 +
.../core/metrics/MetricsFactoryImpl.java | 121 +++--
.../core/astyanax/ColumnNameIteratorTest.java | 7 +-
.../MultiKeyColumnNameIteratorTest.java | 7 +-
.../astyanax/MultiRowColumnIteratorTest.java | 7 +-
stack/corepersistence/graph/pom.xml | 28 +-
.../graph/impl/GraphManagerImpl.java | 288 ++++++++++--
.../usergrid/persistence/map/MapManager.java | 14 +-
.../persistence/map/impl/MapManagerImpl.java | 8 +
.../persistence/map/impl/MapSerialization.java | 9 +
.../map/impl/MapSerializationImpl.java | 130 +++++-
.../persistence/map/MapManagerTest.java | 49 +-
stack/corepersistence/queryindex/pom.xml | 17 +-
.../usergrid/persistence/index/EntityIndex.java | 26 +-
.../persistence/index/IndexBufferConsumer.java | 11 +
.../persistence/index/IndexBufferProducer.java | 1 -
.../usergrid/persistence/index/IndexFig.java | 73 ++-
.../index/IndexOperationMessage.java | 115 ++++-
.../persistence/index/guice/IndexModule.java | 7 +
.../persistence/index/guice/QueueProvider.java | 116 +++++
.../persistence/index/impl/BatchRequest.java | 41 ++
.../persistence/index/impl/BufferQueue.java | 68 +++
.../index/impl/BufferQueueInMemoryImpl.java | 108 +++++
.../index/impl/BufferQueueSQSImpl.java | 307 ++++++++++++
.../persistence/index/impl/DeIndexRequest.java | 115 +++++
.../index/impl/EsEntityIndexBatchImpl.java | 50 +-
.../index/impl/EsEntityIndexImpl.java | 176 ++-----
.../index/impl/EsIndexBufferConsumerImpl.java | 286 ++++++++----
.../index/impl/EsIndexBufferProducerImpl.java | 16 +-
.../persistence/index/impl/EsIndexCache.java | 138 +++---
.../persistence/index/impl/IndexRequest.java | 125 +++++
.../index/guice/TestIndexModule.java | 3 +-
.../index/impl/BufferQueueSQSImplTest.java | 169 +++++++
.../impl/EntityConnectionIndexImplTest.java | 5 +-
.../persistence/index/impl/EntityIndexTest.java | 20 +-
.../persistence/index/impl/EsTestUtils.java | 48 --
.../persistence/queue/QueueManager.java | 4 +-
.../usergrid/persistence/queue/QueueScope.java | 2 +-
.../persistence/queue/QueueScopeFactory.java | 34 --
.../persistence/queue/guice/QueueModule.java | 17 +-
.../queue/impl/QueueScopeFactoryImpl.java | 48 --
.../persistence/queue/impl/QueueScopeImpl.java | 27 +-
.../queue/impl/SQSQueueManagerImpl.java | 214 +++++----
.../persistence/queue/NoAWSCredsRule.java | 98 ++++
.../persistence/queue/QueueManagerTest.java | 29 +-
stack/pom.xml | 43 --
stack/rest/pom.xml | 4 -
.../org/apache/usergrid/rest/RootResource.java | 6 -
stack/services/pom.xml | 13 +-
.../cassandra/ManagementServiceImpl.java | 9 +-
.../services/AbstractCollectionService.java | 40 +-
.../services/AbstractConnectionsService.java | 39 +-
.../notifications/NotificationsService.java | 50 +-
.../services/notifications/QueueListener.java | 5 +-
.../usergrid/services/queues/QueueListener.java | 5 +-
.../notifications/NotifiersServiceIT.java | 6 +
stack/test-utils/pom.xml | 5 -
105 files changed, 3849 insertions(+), 1571 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7b5d2224/stack/awscluster/src/main/dist/init_instance/init_rest_server.sh
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7b5d2224/stack/core/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7b5d2224/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
index cf927ce,161037b..5da7067
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
@@@ -33,21 -30,17 +30,15 @@@ import org.apache.usergrid.persistence.
import org.apache.usergrid.persistence.collection.event.EntityVersionCreated;
import org.apache.usergrid.persistence.collection.event.EntityVersionDeleted;
import org.apache.usergrid.persistence.collection.guice.CollectionModule;
- import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
import org.apache.usergrid.persistence.core.guice.CommonModule;
import org.apache.usergrid.persistence.core.migration.data.DataMigration;
- import org.apache.usergrid.persistence.core.migration.data.MigrationDataProvider;
- import org.apache.usergrid.persistence.core.migration.data.MigrationPlugin;
import org.apache.usergrid.persistence.graph.guice.GraphModule;
- import org.apache.usergrid.persistence.graph.serialization.impl.migration.GraphNode;
import org.apache.usergrid.persistence.index.guice.IndexModule;
-import org.apache.usergrid.persistence.index.impl.BufferQueue;
-import org.apache.usergrid.persistence.index.impl.BufferQueueSQSImpl;
import org.apache.usergrid.persistence.map.guice.MapModule;
import org.apache.usergrid.persistence.queue.guice.QueueModule;
-
- import com.google.inject.AbstractModule;
- import com.google.inject.Provider;
- import com.google.inject.TypeLiteral;
- import com.google.inject.multibindings.Multibinder;
+ import org.slf4j.Logger;
+ import org.slf4j.LoggerFactory;
+ import org.springframework.context.ApplicationContext;
/**
@@@ -76,31 -69,11 +67,31 @@@ public class CoreModule extends Abstra
bind(EntityManagerFactory.class).toProvider( lazyEntityManagerFactoryProvider );
install( new CommonModule());
- install(new CollectionModule());
- install(new GraphModule());
- install( new IndexModule() );
-// install(new MapModule()); TODO, re-enable when index module doesn't depend on queue
-// install(new QueueModule());
+ install( new CollectionModule() {
+ /**
+ * configure our migration data provider for all entities in the system
+ */
+ @Override
+ public void configureMigrationProvider() {
+
+ bind(new TypeLiteral< MigrationDataProvider<EntityIdScope>>(){}).to(
+ AllEntitiesInSystemImpl.class );
+ }
+ } );
+ install( new GraphModule() {
+
+ /**
+ * Override the observable that needs to be used for migration
+ */
+ @Override
+ public void configureMigrationProvider() {
+ bind( new TypeLiteral<MigrationDataProvider<GraphNode>>() {} ).to(
+ AllNodesInGraphImpl.class );
+ }
+ } );
+ install(new IndexModule());
- install(new MapModule());
- install(new QueueModule());
++ // install(new MapModule()); TODO, re-enable when index module doesn't depend on queue
++ // install(new QueueModule());
bind(ManagerCache.class).to( CpManagerCache.class );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7b5d2224/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7b5d2224/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java
index 65dacee,01848cb..95ff9dc
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java
@@@ -47,18 -53,18 +54,20 @@@ import rx.schedulers.Schedulers
public class EntityVersionDeletedHandler implements EntityVersionDeleted {
private static final Logger logger = LoggerFactory.getLogger(EntityVersionDeletedHandler.class );
- @Inject
- private SerializationFig serializationFig;
+
+
+
+ private final EntityManagerFactory emf;
@Inject
- private EntityManagerFactory emf;
+ public EntityVersionDeletedHandler( final EntityManagerFactory emf ) {this.emf = emf;}
+
@Override
- public void versionDeleted(
- final CollectionScope scope, final Id entityId, final List<MvccEntity> entityVersions) {
+ public void versionDeleted( final CollectionScope scope, final Id entityId,
+ final List<MvccLogEntry> entityVersions ) {
+
// This check is for testing purposes and for a test that to be able to dynamically turn
// off and on delete previous versions so that it can test clean-up on read.
@@@ -87,19 -89,18 +92,19 @@@
scope.getName()
);
- rx.Observable.from( entityVersions )
- .buffer(serializationFig.getBufferSize())
- .map(new Func1<List<MvccLogEntry>, List<MvccLogEntry>>() {
+ Observable.from( entityVersions )
+ .collect( ei.createBatch(), new Action2<EntityIndexBatch, MvccEntity>() {
@Override
- public List<MvccLogEntry> call(List<MvccLogEntry> entityList) {
- for (MvccLogEntry entity : entityList) {
- eibatch.deindex(indexScope, entityId, entity.getVersion());
- }
- eibatch.execute();
- return entityList;
+ public void call( final EntityIndexBatch entityIndexBatch, final MvccEntity mvccEntity ) {
+ entityIndexBatch.deindex( indexScope, mvccEntity.getId(), mvccEntity.getVersion() );
}
- }).toBlocking().last();
+ } ).doOnNext( new Action1<EntityIndexBatch>() {
+ @Override
+ public void call( final EntityIndexBatch entityIndexBatch ) {
+ entityIndexBatch.execute();
+ }
+ } ).toBlocking().last();
}
+
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7b5d2224/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7b5d2224/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7b5d2224/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/cache/CachedEntityCollectionManager.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7b5d2224/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/guice/CollectionModule.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/guice/CollectionModule.java
index d5478d4,a73d7a7..68ddab5
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/guice/CollectionModule.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/guice/CollectionModule.java
@@@ -21,19 -21,17 +21,14 @@@ package org.apache.usergrid.persistence
import org.safehaus.guicyfig.GuicyFigModule;
import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
- import org.apache.usergrid.persistence.collection.EntityDeletedFactory;
- import org.apache.usergrid.persistence.collection.EntityVersionCleanupFactory;
- import org.apache.usergrid.persistence.collection.EntityVersionCreatedFactory;
- import org.apache.usergrid.persistence.collection.MvccEntity;
+ import org.apache.usergrid.persistence.collection.impl.EntityVersionTaskFactory;
-import org.apache.usergrid.persistence.collection.MvccEntity;
import org.apache.usergrid.persistence.collection.cache.EntityCacheFig;
import org.apache.usergrid.persistence.collection.event.EntityDeleted;
import org.apache.usergrid.persistence.collection.event.EntityVersionCreated;
import org.apache.usergrid.persistence.collection.event.EntityVersionDeleted;
import org.apache.usergrid.persistence.collection.impl.EntityCollectionManagerFactoryImpl;
--import org.apache.usergrid.persistence.collection.mvcc.MvccLogEntrySerializationStrategy;
import org.apache.usergrid.persistence.collection.mvcc.changelog.ChangeLogGenerator;
import org.apache.usergrid.persistence.collection.mvcc.changelog.ChangeLogGeneratorImpl;
--import org.apache.usergrid.persistence.collection.mvcc.stage.write.WriteStart;
import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
import org.apache.usergrid.persistence.collection.serialization.impl.SerializationModule;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7b5d2224/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerFactoryImpl.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerFactoryImpl.java
index 10d85f8,23e375d..ac82181
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerFactoryImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerFactoryImpl.java
@@@ -26,25 -25,25 +26,18 @@@ import java.util.concurrent.ExecutionEx
import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.collection.EntityCollectionManager;
import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
+ import org.apache.usergrid.persistence.collection.EntityCollectionManagerSync;
+import org.apache.usergrid.persistence.collection.EntityDeletedFactory;
+import org.apache.usergrid.persistence.collection.EntityVersionCleanupFactory;
+import org.apache.usergrid.persistence.collection.EntityVersionCreatedFactory;
import org.apache.usergrid.persistence.collection.cache.CachedEntityCollectionManager;
import org.apache.usergrid.persistence.collection.cache.EntityCacheFig;
--import org.apache.usergrid.persistence.collection.guice.CollectionTaskExecutor;
- import org.apache.usergrid.persistence.collection.mvcc.MvccLogEntrySerializationStrategy;
- import org.apache.usergrid.persistence.collection.mvcc.stage.delete.MarkCommit;
- import org.apache.usergrid.persistence.collection.mvcc.stage.delete.MarkStart;
-import org.apache.usergrid.persistence.collection.guice.Write;
-import org.apache.usergrid.persistence.collection.guice.WriteUpdate;
+ import org.apache.usergrid.persistence.collection.mvcc.MvccEntitySerializationStrategy;
-import org.apache.usergrid.persistence.collection.mvcc.MvccLogEntrySerializationStrategy;
-import org.apache.usergrid.persistence.collection.mvcc.stage.delete.MarkCommit;
-import org.apache.usergrid.persistence.collection.mvcc.stage.delete.MarkStart;
import org.apache.usergrid.persistence.collection.mvcc.stage.write.RollbackAction;
import org.apache.usergrid.persistence.collection.mvcc.stage.write.WriteCommit;
import org.apache.usergrid.persistence.collection.mvcc.stage.write.WriteOptimisticVerify;
import org.apache.usergrid.persistence.collection.mvcc.stage.write.WriteStart;
import org.apache.usergrid.persistence.collection.mvcc.stage.write.WriteUniqueVerify;
- import org.apache.usergrid.persistence.collection.serialization.MvccEntitySerializationStrategy;
- import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
--import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
--import org.apache.usergrid.persistence.core.guice.ProxyImpl;
-import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
--import org.apache.usergrid.persistence.core.task.TaskExecutor;
import com.google.common.base.Preconditions;
import com.google.common.cache.CacheBuilder;
@@@ -74,25 -74,22 +67,24 @@@ public class EntityCollectionManagerFac
private final UniqueValueSerializationStrategy uniqueValueSerializationStrategy;
private final MvccLogEntrySerializationStrategy mvccLogEntrySerializationStrategy;
private final Keyspace keyspace;
- private final EntityVersionCleanupFactory entityVersionCleanupFactory;
- private final EntityVersionCreatedFactory entityVersionCreatedFactory;
- private final EntityDeletedFactory entityDeletedFactory;
+ private final EntityVersionTaskFactory entityVersionTaskFactory;
private final TaskExecutor taskExecutor;
+ private final EntityCacheFig entityCacheFig;
+ private final MetricsFactory metricsFactory;
+ private EntityCacheFig entityCacheFig;
+ private SerializationFig serializationFig;
private LoadingCache<CollectionScope, EntityCollectionManager> ecmCache =
CacheBuilder.newBuilder().maximumSize( 1000 )
.build( new CacheLoader<CollectionScope, EntityCollectionManager>() {
public EntityCollectionManager load( CollectionScope scope ) {
//create the target EM that will perform logic
- final EntityCollectionManager target = new EntityCollectionManagerImpl( writeStart, writeUpdate, writeVerifyUnique,
+ final EntityCollectionManager target = new EntityCollectionManagerImpl(
+ writeStart, writeVerifyUnique,
writeOptimisticVerify, writeCommit, rollback, markStart, markCommit,
entitySerializationStrategy, uniqueValueSerializationStrategy,
- mvccLogEntrySerializationStrategy, keyspace, serializationFig,entityVersionCleanupFactory,
- entityVersionCreatedFactory, entityDeletedFactory, taskExecutor, scope );
- mvccLogEntrySerializationStrategy, keyspace, entityVersionTaskFactory,
- taskExecutor, scope, metricsFactory );
++ mvccLogEntrySerializationStrategy, keyspace, serializationFig,entityVersionTaskFactory, taskExecutor, scope, metricsFactory );
final EntityCollectionManager proxy = new CachedEntityCollectionManager(entityCacheFig, target );
@@@ -113,14 -111,13 +105,12 @@@
final UniqueValueSerializationStrategy uniqueValueSerializationStrategy,
final MvccLogEntrySerializationStrategy mvccLogEntrySerializationStrategy,
final Keyspace keyspace,
- final EntityVersionCleanupFactory entityVersionCleanupFactory,
- final EntityVersionCreatedFactory entityVersionCreatedFactory,
- final EntityDeletedFactory entityDeletedFactory,
+ final EntityVersionTaskFactory entityVersionTaskFactory,
@CollectionTaskExecutor final TaskExecutor taskExecutor,
final EntityCacheFig entityCacheFig,
- final SerializationFig serializationFig) {
+ MetricsFactory metricsFactory) {
this.writeStart = writeStart;
- this.writeUpdate = writeUpdate;
this.writeVerifyUnique = writeVerifyUnique;
this.writeOptimisticVerify = writeOptimisticVerify;
this.writeCommit = writeCommit;
@@@ -131,20 -128,21 +121,18 @@@
this.uniqueValueSerializationStrategy = uniqueValueSerializationStrategy;
this.mvccLogEntrySerializationStrategy = mvccLogEntrySerializationStrategy;
this.keyspace = keyspace;
- this.entityVersionCleanupFactory = entityVersionCleanupFactory;
- this.entityVersionCreatedFactory = entityVersionCreatedFactory;
- this.entityDeletedFactory = entityDeletedFactory;
+ this.entityVersionTaskFactory = entityVersionTaskFactory;
this.taskExecutor = taskExecutor;
this.entityCacheFig = entityCacheFig;
- this.serializationFig = serializationFig;
+ this.metricsFactory = metricsFactory;
}
-
-
@Override
- public EntityCollectionManager createCollectionManager( CollectionScope collectionScope ) {
- Preconditions.checkNotNull( collectionScope );
- try {
- return ecmCache.get( collectionScope );
- }
- catch ( ExecutionException ee ) {
- throw new RuntimeException( ee );
+ public EntityCollectionManager createCollectionManager(CollectionScope collectionScope) {
+ Preconditions.checkNotNull(collectionScope);
+ try{
+ return ecmCache.get(collectionScope);
+ }catch (ExecutionException ee){
+ throw new RuntimeException(ee);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7b5d2224/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
index 19e8937,6ba4513..001e0db
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
@@@ -35,6 -34,9 +37,10 @@@ import org.apache.usergrid.persistence.
import org.apache.usergrid.persistence.collection.MvccEntity;
import org.apache.usergrid.persistence.collection.VersionSet;
import org.apache.usergrid.persistence.collection.guice.CollectionTaskExecutor;
++import org.apache.usergrid.persistence.collection.guice.CollectionTaskExecutor;
+ import org.apache.usergrid.persistence.collection.guice.Write;
+ import org.apache.usergrid.persistence.collection.guice.WriteUpdate;
+ import org.apache.usergrid.persistence.collection.mvcc.MvccEntitySerializationStrategy;
import org.apache.usergrid.persistence.collection.mvcc.MvccLogEntrySerializationStrategy;
import org.apache.usergrid.persistence.collection.mvcc.entity.MvccValidationUtils;
import org.apache.usergrid.persistence.collection.mvcc.stage.CollectionIoEvent;
@@@ -51,8 -51,9 +57,11 @@@ import org.apache.usergrid.persistence.
import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
import org.apache.usergrid.persistence.collection.serialization.UniqueValueSet;
import org.apache.usergrid.persistence.core.guice.ProxyImpl;
+import org.apache.usergrid.persistence.core.task.Task;
+import org.apache.usergrid.persistence.core.task.TaskExecutor;
+ import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
+ import org.apache.usergrid.persistence.core.task.Task;
+ import org.apache.usergrid.persistence.core.task.TaskExecutor;
import org.apache.usergrid.persistence.core.util.Health;
import org.apache.usergrid.persistence.core.util.ValidationUtils;
import org.apache.usergrid.persistence.model.entity.Entity;
@@@ -69,9 -72,14 +80,17 @@@ import com.netflix.astyanax.connectionp
import com.netflix.astyanax.model.ColumnFamily;
import com.netflix.astyanax.model.CqlResult;
import com.netflix.astyanax.serializers.StringSerializer;
++import org.apache.usergrid.persistence.collection.EntityDeletedFactory;
++import org.apache.usergrid.persistence.collection.EntityVersionCleanupFactory;
++import org.apache.usergrid.persistence.collection.EntityVersionCreatedFactory;
+ import org.apache.usergrid.persistence.collection.guice.CollectionTaskExecutor;
+ import org.apache.usergrid.persistence.core.task.Task;
+ import org.apache.usergrid.persistence.core.task.TaskExecutor;
+ import rx.Notification;
import rx.Observable;
import rx.Subscriber;
+ import rx.functions.Action0;
import rx.functions.Action1;
import rx.functions.Func1;
import rx.schedulers.Schedulers;
@@@ -288,6 -368,140 +375,89 @@@ public class EntityCollectionManagerImp
} );
}
+
+ /**
+ * Retrieves all entities that correspond to each field given in the Collection.
+ * @param fields
+ * @return
+ */
+ @Override
+ public Observable<FieldSet> getEntitiesFromFields( final Collection<Field> fields ) {
+ return rx.Observable.just(fields).map( new Func1<Collection<Field>, FieldSet>() {
+ @Override
+ public FieldSet call( Collection<Field> fields ) {
+ try {
+
+ final UUID startTime = UUIDGenerator.newTimeUUID();
+
+ //Get back set of unique values that correspond to collection of fields
+ UniqueValueSet set = uniqueValueSerializationStrategy.load( collectionScope, fields );
+
+ //Short circut if we don't have any uniqueValues from the given fields.
+ if(!set.iterator().hasNext()){
+ return new MutableFieldSet( 0 );
+ }
+
+
+ //loop through each field, and construct an entity load
+ List<Id> entityIds = new ArrayList<>(fields.size());
+ List<UniqueValue> uniqueValues = new ArrayList<>(fields.size());
+
+ for(final Field expectedField: fields) {
+
+ UniqueValue value = set.getValue(expectedField.getName());
+
+ if(value ==null){
+ logger.debug( "Field does not correspond to a unique value" );
+ }
+
+ entityIds.add(value.getEntityId());
+ uniqueValues.add(value);
+ }
+
+ //Load a entity for each entityId we retrieved.
+ final EntitySet entitySet = entitySerializationStrategy.load(collectionScope, entityIds, startTime);
+
+ //now loop through and ensure the entities are there.
+ final MutationBatch deleteBatch = keyspace.prepareMutationBatch();
+
+ final MutableFieldSet response = new MutableFieldSet(fields.size());
+
+ for(final UniqueValue expectedUnique: uniqueValues) {
+ final MvccEntity entity = entitySet.getEntity(expectedUnique.getEntityId());
+
+ //bad unique value, delete this, it's inconsistent
+ if(entity == null || !entity.getEntity().isPresent()){
+ final MutationBatch valueDelete = uniqueValueSerializationStrategy.delete(collectionScope, expectedUnique);
+ deleteBatch.mergeShallow(valueDelete);
+ continue;
+ }
+
+
+ //else add it to our result set
+ response.addEntity(expectedUnique.getField(),entity);
+
+ }
+
+ //TODO: explore making this an Async process
+ //We'll repair it again if we have to
+ deleteBatch.execute();
+
+ return response;
+
+
+ }
+ catch ( ConnectionException e ) {
+ logger.error( "Failed to getIdField", e );
+ throw new RuntimeException( e );
+ }
+ }
+ } );
+ }
+
+
+
- @Override
- public Observable<Entity> update( final Entity entity ) {
-
- logger.debug( "Starting update process" );
-
- //do our input validation
- Preconditions.checkNotNull( entity, "Entity is required in the new stage of the mvcc write" );
-
- final Id entityId = entity.getId();
-
-
- ValidationUtils.verifyIdentity( entityId );
-
- // create our observable and start the write
- CollectionIoEvent<Entity> writeData = new CollectionIoEvent<Entity>( collectionScope, entity );
-
-
- Observable<CollectionIoEvent<MvccEntity>> observable = stageRunner( writeData, writeUpdate );
-
-
- final Timer.Context timer = updateTimer.time();
- return observable.map( writeCommit ).doOnNext(new Action1<Entity>() {
- @Override
- public void call(final Entity entity) {
- logger.debug("sending entity to the queue");
-
- //we an update, signal the fix
- taskExecutor.submit( entityVersionTaskFactory.getCreatedTask( collectionScope, entity ));
-
- taskExecutor.submit( entityVersionTaskFactory.getCleanupTask( collectionScope, entity.getId(), entity.getVersion(), false) );
- //TODO T.N Change this to fire a task
- // Observable.from( new CollectionIoEvent<Id>(collectionScope,
- // entityId ) ).map( load ).subscribeOn( Schedulers.io() ).subscribe();
-
-
- }
- }).doOnError(rollback)
- .doOnNext(new Action1<Entity>() {
- @Override
- public void call(Entity entity) {
- updateMeter.mark();
- }
- })
- .doOnCompleted(new Action0() {
- @Override
- public void call() {
- timer.stop();
- }
- });
- }
-
+
// fire the stages
public Observable<CollectionIoEvent<MvccEntity>> stageRunner( CollectionIoEvent<Entity> writeData,
WriteStart writeState ) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7b5d2224/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityDeletedTask.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityDeletedTask.java
index dbe58d4,83f165d..5472645
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityDeletedTask.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityDeletedTask.java
@@@ -22,9 -22,8 +22,8 @@@ import com.google.inject.Inject
import com.google.inject.assistedinject.Assisted;
import com.netflix.astyanax.MutationBatch;
import org.apache.usergrid.persistence.collection.CollectionScope;
- import org.apache.usergrid.persistence.collection.EntityVersionCleanupFactory;
import org.apache.usergrid.persistence.collection.event.EntityDeleted;
-import org.apache.usergrid.persistence.collection.mvcc.MvccEntitySerializationStrategy;
+import org.apache.usergrid.persistence.collection.serialization.MvccEntitySerializationStrategy;
import org.apache.usergrid.persistence.collection.mvcc.MvccLogEntrySerializationStrategy;
import org.apache.usergrid.persistence.core.task.Task;
import org.apache.usergrid.persistence.model.entity.Id;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7b5d2224/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTask.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTask.java
index 7c39ef4,65506ec..27c3db8
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTask.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTask.java
@@@ -20,9 -20,17 +20,18 @@@ package org.apache.usergrid.persistence
import java.util.Iterator;
import java.util.List;
+import java.util.Set;
import java.util.UUID;
+ import com.google.inject.Inject;
+ import com.google.inject.assistedinject.Assisted;
+ import org.apache.usergrid.persistence.collection.MvccEntity;
+ import org.apache.usergrid.persistence.collection.serialization.UniqueValue;
+ import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
+ import org.apache.usergrid.persistence.collection.serialization.impl.UniqueValueImpl;
+ import org.apache.usergrid.persistence.collection.util.EntityUtils;
+ import org.apache.usergrid.persistence.model.entity.Entity;
+ import org.apache.usergrid.persistence.model.field.Field;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@@ -70,6 -75,7 +79,7 @@@ public class EntityVersionCleanupTask i
private final CollectionScope scope;
private final Id entityId;
private final UUID version;
- private final int numToSkip;
++ private final boolean includeVersion;
@Inject
@@@ -89,6 -100,8 +103,8 @@@
this.scope = scope;
this.entityId = entityId;
this.version = version;
+
- numToSkip = includeVersion? 0: 1;
++ includeVersion = includeVersion;
}
@@@ -117,94 -130,79 +133,92 @@@
@Override
public Void call() throws Exception {
//TODO Refactor this logic into a a class that can be invoked from anywhere
- //load every entity we have history of
- Observable<List<MvccEntity>> deleteFieldsObservable =
- Observable.create(new ObservableIterator<MvccEntity>("deleteColumns") {
- @Override
- protected Iterator<MvccEntity> getIterator() {
- Iterator<MvccEntity> entities = entitySerializationStrategy.loadDescendingHistory(
- scope, entityId, version, 1000); // TODO: what fetchsize should we use here?
- return entities;
- }
- })
- //buffer them for efficiency
- .skip(numToSkip)
- .buffer(serializationFig.getBufferSize()).doOnNext(
- new Action1<List<MvccEntity>>() {
- @Override
- public void call(final List<MvccEntity> mvccEntities) {
- final MutationBatch batch = keyspace.prepareMutationBatch();
- final MutationBatch entityBatch = keyspace.prepareMutationBatch();
- final MutationBatch logBatch = keyspace.prepareMutationBatch();
--
- for (MvccEntity mvccEntity : mvccEntities) {
- final UUID entityVersion = mvccEntity.getVersion();
--
- //iterate all unique values
-
- //if the entity is present process the fields
- if(mvccEntity.getEntity().isPresent()) {
- final Entity entity = mvccEntity.getEntity().get();
-
- //remove all unique fields from the index
- for ( final Field field : EntityUtils.getUniqueFields(entity )) {
-
- final UniqueValue unique = new UniqueValueImpl( field, entityId, entityVersion );
- final MutationBatch deleteMutation =
- uniqueValueSerializationStrategy.delete( scope, unique );
- batch.mergeShallow( deleteMutation );
++ //iterate all unique values
+ final BlockingObservable<Long> uniqueValueCleanup =
+ Observable.create( new ObservableIterator<UniqueValue>( "Unique value load" ) {
+ @Override
+ protected Iterator<UniqueValue> getIterator() {
+ return uniqueValueSerializationStrategy.getAllUniqueFields( scope, entityId );
+ }
+ } )
+
+ //skip current versions
+ .skipWhile( new Func1<UniqueValue, Boolean>() {
+ @Override
+ public Boolean call( final UniqueValue uniqueValue ) {
+ return version.equals( uniqueValue.getEntityVersion() );
}
+ } )
+ //buffer our buffer size, then roll them all up in a single batch mutation
+ .buffer( serializationFig.getBufferSize() ).doOnNext( new Action1<List<UniqueValue>>() {
+ @Override
+ public void call( final List<UniqueValue> uniqueValues ) {
+ final MutationBatch uniqueCleanupBatch = keyspace.prepareMutationBatch();
+
+
+ for ( UniqueValue value : uniqueValues ) {
+ uniqueCleanupBatch.mergeShallow( uniqueValueSerializationStrategy.delete( scope, value ) );
}
- final MutationBatch entityDelete = entitySerializationStrategy
- .delete(scope, entityId, mvccEntity.getVersion());
- entityBatch.mergeShallow( entityDelete );
- final MutationBatch logDelete = logEntrySerializationStrategy
- .delete(scope, entityId, version);
- logBatch.mergeShallow(logDelete);
+ try {
+ uniqueCleanupBatch.execute();
+ }
+ catch ( ConnectionException e ) {
+ throw new RuntimeException( "Unable to execute batch mutation", e );
+ }
}
+ } ).subscribeOn( Schedulers.io() ).longCount().toBlocking();
- try {
- batch.execute();
- } catch (ConnectionException e1) {
- throw new RuntimeException("Unable to execute " +
- "unique value " +
- "delete", e1);
- }
- fireEvents(mvccEntities);
- try {
- entityBatch.execute();
- } catch (ConnectionException e) {
- throw new RuntimeException("Unable to delete entities in cleanup", e);
+
+ //start calling the listeners for remove log entries
+ BlockingObservable<Long> versionsDeletedObservable =
+
+ Observable.create( new ObservableIterator<MvccLogEntry>( "Log entry iterator" ) {
+ @Override
+ protected Iterator<MvccLogEntry> getIterator() {
+
+ return new LogEntryIterator( logEntrySerializationStrategy, scope, entityId, version,
+ serializationFig.getBufferSize() );
}
+ } )
+ //skip current version
+ .skipWhile( new Func1<MvccLogEntry, Boolean>() {
+ @Override
+ public Boolean call( final MvccLogEntry mvccLogEntry ) {
+ return version.equals( mvccLogEntry.getVersion() );
+ }
+ } )
+ //buffer them for efficiency
+ .buffer( serializationFig.getBufferSize() ).doOnNext( new Action1<List<MvccLogEntry>>() {
+ @Override
+ public void call( final List<MvccLogEntry> mvccEntities ) {
+
+ fireEvents( mvccEntities );
+
+ final MutationBatch logCleanupBatch = keyspace.prepareMutationBatch();
+
+
+ for ( MvccLogEntry entry : mvccEntities ) {
+ logCleanupBatch.mergeShallow( logEntrySerializationStrategy.delete( scope, entityId, entry.getVersion() ));
+ }
- try {
- logBatch.execute();
- } catch (ConnectionException e) {
- throw new RuntimeException("Unable to delete entities from the log", e);
+ try {
+ logCleanupBatch.execute();
+ }
+ catch ( ConnectionException e ) {
+ throw new RuntimeException( "Unable to execute batch mutation", e );
+ }
}
+ } ).subscribeOn( Schedulers.io() ).longCount().toBlocking();
+
+ //wait or this to complete
+ final Long removedCount = uniqueValueCleanup.last();
- }
- }
- );
+ logger.debug( "Removed unique values for {} entities of entity {}", removedCount, entityId );
- final int removedCount = deleteFieldsObservable.count().toBlocking().last();
+ final Long versionCleanupCount = versionsDeletedObservable.last();
- logger.debug("Removed unique values for {} entities of entity {}",removedCount,entityId);
+ logger.debug( "Removed {} previous entity versions of entity {}", versionCleanupCount, entityId );
return null;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7b5d2224/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/UniqueValueSerializationStrategy.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/UniqueValueSerializationStrategy.java
index 5b7898e,60275d9..d7a6407
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/UniqueValueSerializationStrategy.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/UniqueValueSerializationStrategy.java
@@@ -23,9 -22,9 +23,10 @@@ import java.util.Iterator
import com.netflix.astyanax.MutationBatch;
import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
+ import com.netflix.astyanax.model.ConsistencyLevel;
import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.core.migration.schema.Migration;
+import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.field.Field;
@@@ -55,27 -53,23 +56,38 @@@ public interface UniqueValueSerializati
/**
* Load UniqueValue that matches field from collection or null if that value does not exist.
*
- * @param colScope Collection scope in which to look for field name/value
+ * @param collectionScope scope in which to look for field name/value
* @param fields Field name/value to search for
+ *
* @return UniqueValueSet containing fields from the collection that exist in cassandra
+ *
* @throws ConnectionException on error connecting to Cassandra
*/
- public UniqueValueSet load( CollectionScope colScope, Collection<Field> fields ) throws ConnectionException;
+ public UniqueValueSet load( CollectionScope collectionScope, Collection<Field> fields ) throws ConnectionException;
+ /**
- * Load UniqueValue that matches field from collection or null if that value does not exist.
- *
- * @param colScope Collection scope in which to look for field name/value
- * @param consistencyLevel Consistency level of query
- * @param fields Field name/value to search for
- * @return UniqueValueSet containing fields from the collection that exist in cassandra
- * @throws ConnectionException on error connecting to Cassandra
- */
++ * Load UniqueValue that matches field from collection or null if that value does not exist.
++ *
++ * @param colScope Collection scope in which to look for field name/value
++ * @param consistencyLevel Consistency level of query
++ * @param fields Field name/value to search for
++ * @return UniqueValueSet containing fields from the collection that exist in cassandra
++ * @throws ConnectionException on error connecting to Cassandra
++ */
+ public UniqueValueSet load( CollectionScope colScope, ConsistencyLevel consistencyLevel, Collection<Field> fields ) throws ConnectionException;
++
+
+ /**
+ * Loads the currently persisted history of every unique value the entity has held. This will
+ * start from the max version and return values in descending version order. Note that for entities
+ * with more than one unique field, sequential fields can be returned with the same version.
+ * @param collectionScope The scope the entity is stored in
+ * @param entityId
+ * @return
+ */
+ public Iterator<UniqueValue> getAllUniqueFields(CollectionScope collectionScope, Id entityId);
+
+
/**
* Delete the specified Unique Value from Cassandra.
*
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7b5d2224/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
index 7aed8db,47372ae..108f2e8
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
@@@ -38,16 -36,8 +40,17 @@@ import org.apache.usergrid.persistence.
import org.apache.usergrid.persistence.collection.serialization.UniqueValue;
import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
import org.apache.usergrid.persistence.collection.serialization.UniqueValueSet;
+import org.apache.usergrid.persistence.collection.util.EntityUtils;
+import org.apache.usergrid.persistence.core.astyanax.ColumnNameIterator;
+import org.apache.usergrid.persistence.core.astyanax.ColumnParser;
+import org.apache.usergrid.persistence.core.astyanax.ColumnTypes;
+import org.apache.usergrid.persistence.core.astyanax.IdRowCompositeSerializer;
+import org.apache.usergrid.persistence.core.astyanax.MultiTennantColumnFamily;
+import org.apache.usergrid.persistence.core.astyanax.MultiTennantColumnFamilyDefinition;
+import org.apache.usergrid.persistence.core.astyanax.ScopedRowKey;
+ import org.apache.usergrid.persistence.core.migration.schema.Migration;
import org.apache.usergrid.persistence.core.util.ValidationUtils;
+import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.field.Field;
@@@ -78,40 -65,23 +81,43 @@@ public class UniqueValueSerializationSt
private static final EntityVersionSerializer ENTITY_VERSION_SER = new EntityVersionSerializer();
- private static final MultiTennantColumnFamily<ScopedRowKey<CollectionPrefixedKey<Field>>, EntityVersion> CF_UNIQUE_VALUES =
- new MultiTennantColumnFamily<>( "Unique_Values", ROW_KEY_SER,
- ENTITY_VERSION_SER );
+ private static final MultiTennantColumnFamily<ScopedRowKey<CollectionPrefixedKey<Field>>, EntityVersion>
+ CF_UNIQUE_VALUES = new MultiTennantColumnFamily<>( "Unique_Values", ROW_KEY_SER, ENTITY_VERSION_SER );
+
+
++
+ private static final IdRowCompositeSerializer ID_SER = IdRowCompositeSerializer.get();
+
+
+ private static final CollectionScopedRowKeySerializer<Id> ENTITY_ROW_KEY_SER =
+ new CollectionScopedRowKeySerializer<>( ID_SER );
+
+
+ private static final MultiTennantColumnFamily<ScopedRowKey<CollectionPrefixedKey<Id>>, UniqueFieldEntry>
+ CF_ENTITY_UNIQUE_VALUES =
+ new MultiTennantColumnFamily<>( "Entity_Unique_Values", ENTITY_ROW_KEY_SER, UniqueFieldEntrySerializer.get() );
+
+ public static final int COL_VALUE = 0x0;
+
+
- private final Keyspace keyspace;
+ private final SerializationFig serializationFig;
+ protected final Keyspace keyspace;
- private final CassandraFig cassandraFig;
++ private final CassandraFig cassandraFig;
+
/**
* Construct serialization strategy for keyspace.
*
* @param keyspace Keyspace in which to store Unique Values.
+ * @param serializationFig
*/
@Inject
- public UniqueValueSerializationStrategyImpl( final Keyspace keyspace, final SerializationFig serializationFig ) {
- public UniqueValueSerializationStrategyImpl( final Keyspace keyspace, final CassandraFig cassandraFig) {
- this.cassandraFig = cassandraFig;
++ public UniqueValueSerializationStrategyImpl( final Keyspace keyspace, final CassandraFig cassandraFig, final SerializationFig serializationFig ) {
this.keyspace = keyspace;
++ this.cassandraFig = cassandraFig;
+ this.serializationFig = serializationFig;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7b5d2224/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/util/EntityUtils.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/util/EntityUtils.java
index beff3bd,cf964a3..20edb66
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/util/EntityUtils.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/util/EntityUtils.java
@@@ -2,9 -2,10 +2,10 @@@ package org.apache.usergrid.persistence
import java.util.ArrayList;
- import java.util.Collection;
import java.util.List;
-import java.util.Set;
++import java.util.Collection;
import java.util.UUID;
+ import org.apache.usergrid.persistence.model.field.Field;
import org.apache.commons.lang3.reflect.FieldUtils;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7b5d2224/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7b5d2224/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java
index 8f78799,f19d613..246bbca
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java
@@@ -96,7 -126,8 +126,8 @@@ public class GraphManagerImpl implement
final GraphFig graphFig,
final EdgeDeleteListener edgeDeleteListener,
final NodeDeleteListener nodeDeleteListener,
- final ApplicationScope scope) {
- @Assisted final ApplicationScope scope,
++ final ApplicationScope scope,
+ MetricsFactory metricsFactory) {
ValidationUtils.validateApplicationScope( scope );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7b5d2224/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/MapManager.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7b5d2224/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7b5d2224/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7b5d2224/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7b5d2224/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
index 7fccd11,50b994d..27fe705
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
@@@ -19,8 -19,12 +19,9 @@@
package org.apache.usergrid.persistence.index.guice;
- import org.apache.usergrid.persistence.core.guice.CommonModule;
+ import org.apache.usergrid.persistence.collection.guice.CollectionModule;
import org.apache.usergrid.persistence.core.guice.TestModule;
+ import org.apache.usergrid.persistence.core.guice.CommonModule;
-import org.apache.usergrid.persistence.index.impl.BufferQueue;
-import org.apache.usergrid.persistence.index.impl.BufferQueueInMemoryImpl;
-import org.apache.usergrid.persistence.index.impl.BufferQueueSQSImpl;
public class TestIndexModule extends TestModule {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7b5d2224/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
index e4043d2,ca9bf79..9cb5297
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
@@@ -24,10 -24,14 +24,12 @@@ import java.util.*
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.atomic.AtomicLong;
+ import org.apache.usergrid.persistence.core.guice.MigrationManagerRule;
import org.apache.usergrid.persistence.index.*;
-import org.apache.usergrid.persistence.model.field.ArrayField;
-import org.apache.usergrid.persistence.model.field.EntityObjectField;
-import org.apache.usergrid.persistence.model.field.UUIDField;
+import org.apache.usergrid.persistence.model.field.*;
import org.apache.usergrid.persistence.model.field.value.EntityObject;
import org.junit.Ignore;
+ import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.slf4j.Logger;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7b5d2224/stack/pom.xml
----------------------------------------------------------------------
[37/50] [abbrv] incubator-usergrid git commit: Second pass at fixing
the merge
Posted by sf...@apache.org.
Second pass at fixing the merge
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/762e43e1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/762e43e1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/762e43e1
Branch: refs/heads/USERGRID-480
Commit: 762e43e1ee9dfda1a07b9ba6ea7cd13cede06afc
Parents: 7b5d222
Author: Todd Nine <tn...@apigee.com>
Authored: Tue Mar 17 17:26:25 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Tue Mar 17 17:26:25 2015 -0600
----------------------------------------------------------------------
.../usergrid/corepersistence/CoreModule.java | 37 +++++------
.../corepersistence/CpEntityManagerFactory.java | 21 +++----
.../events/EntityVersionDeletedHandler.java | 8 +--
.../collection/EntityCollectionManager.java | 6 --
.../EntityCollectionManagerFactoryImpl.java | 22 ++++---
.../impl/EntityCollectionManagerImpl.java | 65 ++++++++------------
.../impl/EntityVersionCleanupTask.java | 9 +--
.../MvccEntitySerializationStrategyV1Impl.java | 2 +-
.../MvccEntitySerializationStrategyV3Impl.java | 1 -
.../migration/MvccEntityDataMigrationImpl.java | 8 +--
.../collection/EntityCollectionManagerIT.java | 2 +-
.../impl/EntityVersionCleanupTaskTest.java | 13 ++--
...ntitySerializationStrategyProxyV1_3Test.java | 3 +
...ntitySerializationStrategyProxyV2_3Test.java | 3 +
...ccEntitySerializationStrategyV1ImplTest.java | 2 +
.../impl/GraphManagerFactoryImpl.java | 17 ++---
16 files changed, 104 insertions(+), 115 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/762e43e1/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
index 5da7067..2e9b780 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
@@ -15,30 +15,37 @@
*/
package org.apache.usergrid.corepersistence;
-import com.google.inject.AbstractModule;
-import com.google.inject.Provider;
-import com.google.inject.multibindings.Multibinder;
-import org.apache.usergrid.corepersistence.migration.EntityDataMigration;
-import org.apache.usergrid.corepersistence.migration.EntityTypeMappingMigration;
-import org.apache.usergrid.corepersistence.migration.GraphShardVersionMigration;
+import org.springframework.context.ApplicationContext;
+
import org.apache.usergrid.corepersistence.events.EntityDeletedHandler;
import org.apache.usergrid.corepersistence.events.EntityVersionCreatedHandler;
import org.apache.usergrid.corepersistence.events.EntityVersionDeletedHandler;
+import org.apache.usergrid.corepersistence.migration.CoreMigration;
+import org.apache.usergrid.corepersistence.migration.CoreMigrationPlugin;
+import org.apache.usergrid.corepersistence.migration.EntityTypeMappingMigration;
+import org.apache.usergrid.corepersistence.migration.MigrationModuleVersionPlugin;
+import org.apache.usergrid.corepersistence.rx.impl.AllApplicationsObservableImpl;
+import org.apache.usergrid.corepersistence.rx.impl.AllEntitiesInSystemImpl;
+import org.apache.usergrid.corepersistence.rx.impl.AllNodesInGraphImpl;
import org.apache.usergrid.persistence.EntityManagerFactory;
import org.apache.usergrid.persistence.collection.event.EntityDeleted;
import org.apache.usergrid.persistence.collection.event.EntityVersionCreated;
import org.apache.usergrid.persistence.collection.event.EntityVersionDeleted;
import org.apache.usergrid.persistence.collection.guice.CollectionModule;
+import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
import org.apache.usergrid.persistence.core.guice.CommonModule;
import org.apache.usergrid.persistence.core.migration.data.DataMigration;
+import org.apache.usergrid.persistence.core.migration.data.MigrationDataProvider;
+import org.apache.usergrid.persistence.core.migration.data.MigrationPlugin;
import org.apache.usergrid.persistence.graph.guice.GraphModule;
+import org.apache.usergrid.persistence.graph.serialization.impl.migration.GraphNode;
import org.apache.usergrid.persistence.index.guice.IndexModule;
-import org.apache.usergrid.persistence.map.guice.MapModule;
-import org.apache.usergrid.persistence.queue.guice.QueueModule;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.context.ApplicationContext;
+
+import com.google.inject.AbstractModule;
+import com.google.inject.Provider;
+import com.google.inject.TypeLiteral;
+import com.google.inject.multibindings.Multibinder;
/**
@@ -74,7 +81,7 @@ public class CoreModule extends AbstractModule {
@Override
public void configureMigrationProvider() {
- bind(new TypeLiteral< MigrationDataProvider<EntityIdScope>>(){}).to(
+ bind(new TypeLiteral<MigrationDataProvider<EntityIdScope>>(){}).to(
AllEntitiesInSystemImpl.class );
}
} );
@@ -95,12 +102,6 @@ public class CoreModule extends AbstractModule {
bind(ManagerCache.class).to( CpManagerCache.class );
- Multibinder<DataMigration> dataMigrationMultibinder =
- Multibinder.newSetBinder( binder(), DataMigration.class );
- dataMigrationMultibinder.addBinding().to( EntityTypeMappingMigration.class );
- dataMigrationMultibinder.addBinding().to( GraphShardVersionMigration.class );
- dataMigrationMultibinder.addBinding().to( EntityDataMigration.class );
-
Multibinder<EntityDeleted> entityBinder =
Multibinder.newSetBinder(binder(), EntityDeleted.class);
entityBinder.addBinding().to(EntityDeletedHandler.class);
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/762e43e1/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index 01dfa1f..f76b9fc 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -15,15 +15,14 @@
*/
package org.apache.usergrid.corepersistence;
-import com.google.common.base.Optional;
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheLoader;
-import com.google.common.cache.LoadingCache;
-import com.google.inject.Guice;
-import com.google.inject.Injector;
-import static java.lang.String.CASE_INSENSITIVE_ORDER;
-import java.util.*;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.UUID;
import java.util.concurrent.atomic.AtomicBoolean;
import org.slf4j.Logger;
@@ -48,9 +47,8 @@ import org.apache.usergrid.persistence.cassandra.Setup;
import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.collection.EntityCollectionManager;
import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl;
-import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
-import org.apache.usergrid.persistence.core.migration.data.DataMigrationManager;
+import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.apache.usergrid.persistence.core.migration.data.MigrationDataProvider;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
@@ -59,7 +57,6 @@ import org.apache.usergrid.persistence.entities.Application;
import org.apache.usergrid.persistence.exceptions.ApplicationAlreadyExistsException;
import org.apache.usergrid.persistence.exceptions.DuplicateUniquePropertyExistsException;
import org.apache.usergrid.persistence.exceptions.EntityNotFoundException;
-import org.apache.usergrid.persistence.exceptions.OrganizationAlreadyExistsException;
import org.apache.usergrid.persistence.graph.Edge;
import org.apache.usergrid.persistence.graph.GraphManager;
import org.apache.usergrid.persistence.graph.SearchByEdgeType;
@@ -71,13 +68,13 @@ import org.apache.usergrid.persistence.model.entity.SimpleId;
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
import org.apache.usergrid.utils.UUIDUtils;
+import com.google.common.base.Optional;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import com.google.inject.Injector;
import com.google.inject.Key;
import com.google.inject.TypeLiteral;
-import com.yammer.metrics.annotation.Metered;
import rx.Observable;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/762e43e1/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java
index 95ff9dc..c45949b 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java
@@ -85,7 +85,7 @@ public class EntityVersionDeletedHandler implements EntityVersionDeleted {
CpEntityManagerFactory cpemf = (CpEntityManagerFactory)emf;
- final EntityIndex ei = cpemf.getManagerCache().getEntityIndex(scope);
+ final EntityIndex ei = cpemf.getManagerCache().getEntityIndex( scope );
final IndexScope indexScope = new IndexScopeImpl(
new SimpleId(scope.getOwner().getUuid(), scope.getOwner().getType()),
@@ -93,10 +93,10 @@ public class EntityVersionDeletedHandler implements EntityVersionDeleted {
);
Observable.from( entityVersions )
- .collect( ei.createBatch(), new Action2<EntityIndexBatch, MvccEntity>() {
+ .collect( ei.createBatch(), new Action2<EntityIndexBatch, MvccLogEntry>() {
@Override
- public void call( final EntityIndexBatch entityIndexBatch, final MvccEntity mvccEntity ) {
- entityIndexBatch.deindex( indexScope, mvccEntity.getId(), mvccEntity.getVersion() );
+ public void call( final EntityIndexBatch entityIndexBatch, final MvccLogEntry mvccLogEntry ) {
+ entityIndexBatch.deindex( indexScope, mvccLogEntry.getEntityId(), mvccLogEntry.getVersion() );
}
} ).doOnNext( new Action1<EntityIndexBatch>() {
@Override
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/762e43e1/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityCollectionManager.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityCollectionManager.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityCollectionManager.java
index 35fc5d4..6532ee6 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityCollectionManager.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityCollectionManager.java
@@ -75,12 +75,6 @@ public interface EntityCollectionManager {
*/
public Observable<EntitySet> load(Collection<Id> entityIds);
- /**
- * Takes the change and reloads an entity with all changes applied in this entity applied.
- * The resulting entity from calling load will be the previous version of this entity plus
- * the entity in this object applied to it.
- */
- public Observable<Entity> update ( Entity entity );
/**
* Returns health of entity data store.
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/762e43e1/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerFactoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerFactoryImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerFactoryImpl.java
index ac82181..409467c 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerFactoryImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerFactoryImpl.java
@@ -26,18 +26,23 @@ import java.util.concurrent.ExecutionException;
import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.collection.EntityCollectionManager;
import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
-import org.apache.usergrid.persistence.collection.EntityCollectionManagerSync;
-import org.apache.usergrid.persistence.collection.EntityDeletedFactory;
-import org.apache.usergrid.persistence.collection.EntityVersionCleanupFactory;
-import org.apache.usergrid.persistence.collection.EntityVersionCreatedFactory;
import org.apache.usergrid.persistence.collection.cache.CachedEntityCollectionManager;
import org.apache.usergrid.persistence.collection.cache.EntityCacheFig;
-import org.apache.usergrid.persistence.collection.mvcc.MvccEntitySerializationStrategy;
+import org.apache.usergrid.persistence.collection.guice.CollectionTaskExecutor;
+import org.apache.usergrid.persistence.collection.mvcc.MvccLogEntrySerializationStrategy;
+import org.apache.usergrid.persistence.collection.mvcc.stage.delete.MarkCommit;
+import org.apache.usergrid.persistence.collection.mvcc.stage.delete.MarkStart;
import org.apache.usergrid.persistence.collection.mvcc.stage.write.RollbackAction;
import org.apache.usergrid.persistence.collection.mvcc.stage.write.WriteCommit;
import org.apache.usergrid.persistence.collection.mvcc.stage.write.WriteOptimisticVerify;
import org.apache.usergrid.persistence.collection.mvcc.stage.write.WriteStart;
import org.apache.usergrid.persistence.collection.mvcc.stage.write.WriteUniqueVerify;
+import org.apache.usergrid.persistence.collection.serialization.MvccEntitySerializationStrategy;
+import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
+import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
+import org.apache.usergrid.persistence.core.guice.ProxyImpl;
+import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
+import org.apache.usergrid.persistence.core.task.TaskExecutor;
import com.google.common.base.Preconditions;
import com.google.common.cache.CacheBuilder;
@@ -45,6 +50,7 @@ import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import com.google.inject.Inject;
import com.google.inject.Singleton;
+import com.google.inject.assistedinject.Assisted;
import com.netflix.astyanax.Keyspace;
@@ -72,25 +78,21 @@ public class EntityCollectionManagerFactoryImpl implements EntityCollectionManag
private final EntityCacheFig entityCacheFig;
private final MetricsFactory metricsFactory;
- private EntityCacheFig entityCacheFig;
- private SerializationFig serializationFig;
private LoadingCache<CollectionScope, EntityCollectionManager> ecmCache =
CacheBuilder.newBuilder().maximumSize( 1000 )
.build( new CacheLoader<CollectionScope, EntityCollectionManager>() {
public EntityCollectionManager load( CollectionScope scope ) {
-
//create the target EM that will perform logic
final EntityCollectionManager target = new EntityCollectionManagerImpl(
writeStart, writeVerifyUnique,
writeOptimisticVerify, writeCommit, rollback, markStart, markCommit,
entitySerializationStrategy, uniqueValueSerializationStrategy,
- mvccLogEntrySerializationStrategy, keyspace, serializationFig,entityVersionTaskFactory, taskExecutor, scope, metricsFactory );
+ mvccLogEntrySerializationStrategy, keyspace,entityVersionTaskFactory, taskExecutor, scope, metricsFactory );
final EntityCollectionManager proxy = new CachedEntityCollectionManager(entityCacheFig, target );
return proxy;
-// return target;
}
} );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/762e43e1/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
index 001e0db..f565fab 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
@@ -19,28 +19,22 @@
package org.apache.usergrid.persistence.collection.impl;
-import java.util.*;
-
-import com.netflix.astyanax.MutationBatch;
-import org.apache.usergrid.persistence.collection.*;
-import org.apache.usergrid.persistence.collection.serialization.impl.MutableFieldSet;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.UUID;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.collection.EntityCollectionManager;
-import org.apache.usergrid.persistence.collection.EntityDeletedFactory;
import org.apache.usergrid.persistence.collection.EntitySet;
-import org.apache.usergrid.persistence.collection.EntityVersionCleanupFactory;
-import org.apache.usergrid.persistence.collection.EntityVersionCreatedFactory;
+import org.apache.usergrid.persistence.collection.FieldSet;
import org.apache.usergrid.persistence.collection.MvccEntity;
import org.apache.usergrid.persistence.collection.VersionSet;
import org.apache.usergrid.persistence.collection.guice.CollectionTaskExecutor;
-import org.apache.usergrid.persistence.collection.guice.CollectionTaskExecutor;
-import org.apache.usergrid.persistence.collection.guice.Write;
-import org.apache.usergrid.persistence.collection.guice.WriteUpdate;
-import org.apache.usergrid.persistence.collection.mvcc.MvccEntitySerializationStrategy;
import org.apache.usergrid.persistence.collection.mvcc.MvccLogEntrySerializationStrategy;
import org.apache.usergrid.persistence.collection.mvcc.entity.MvccValidationUtils;
import org.apache.usergrid.persistence.collection.mvcc.stage.CollectionIoEvent;
@@ -52,13 +46,11 @@ import org.apache.usergrid.persistence.collection.mvcc.stage.write.WriteOptimist
import org.apache.usergrid.persistence.collection.mvcc.stage.write.WriteStart;
import org.apache.usergrid.persistence.collection.mvcc.stage.write.WriteUniqueVerify;
import org.apache.usergrid.persistence.collection.serialization.MvccEntitySerializationStrategy;
-import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
import org.apache.usergrid.persistence.collection.serialization.UniqueValue;
import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
import org.apache.usergrid.persistence.collection.serialization.UniqueValueSet;
+import org.apache.usergrid.persistence.collection.serialization.impl.MutableFieldSet;
import org.apache.usergrid.persistence.core.guice.ProxyImpl;
-import org.apache.usergrid.persistence.core.task.Task;
-import org.apache.usergrid.persistence.core.task.TaskExecutor;
import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.apache.usergrid.persistence.core.task.Task;
import org.apache.usergrid.persistence.core.task.TaskExecutor;
@@ -75,17 +67,12 @@ import com.google.common.base.Preconditions;
import com.google.inject.Inject;
import com.google.inject.assistedinject.Assisted;
import com.netflix.astyanax.Keyspace;
+import com.netflix.astyanax.MutationBatch;
import com.netflix.astyanax.connectionpool.OperationResult;
import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
import com.netflix.astyanax.model.ColumnFamily;
import com.netflix.astyanax.model.CqlResult;
import com.netflix.astyanax.serializers.StringSerializer;
-import org.apache.usergrid.persistence.collection.EntityDeletedFactory;
-import org.apache.usergrid.persistence.collection.EntityVersionCleanupFactory;
-import org.apache.usergrid.persistence.collection.EntityVersionCreatedFactory;
-import org.apache.usergrid.persistence.collection.guice.CollectionTaskExecutor;
-import org.apache.usergrid.persistence.core.task.Task;
-import org.apache.usergrid.persistence.core.task.TaskExecutor;
import rx.Notification;
import rx.Observable;
@@ -224,19 +211,19 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
entity.getVersion(), false ));
//post-processing to come later. leave it empty for now.
}
- }).doOnError(rollback)
- .doOnEach(new Action1<Notification<? super Entity>>() {
+ }).doOnError( rollback )
+ .doOnEach( new Action1<Notification<? super Entity>>() {
@Override
- public void call(Notification<? super Entity> notification) {
+ public void call( Notification<? super Entity> notification ) {
writeMeter.mark();
}
- })
- .doOnCompleted(new Action0() {
+ } )
+ .doOnCompleted( new Action0() {
@Override
public void call() {
timer.stop();
}
- });
+ } );
}
@@ -250,7 +237,7 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
final Timer.Context timer = deleteTimer.time();
Observable<Id> o = Observable.from(new CollectionIoEvent<Id>(collectionScope, entityId))
.map(markStart)
- .doOnNext(markCommit)
+ .doOnNext( markCommit )
.map(new Func1<CollectionIoEvent<MvccEntity>, Id>() {
@Override
@@ -269,12 +256,12 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
deleteMeter.mark();
}
})
- .doOnCompleted(new Action0() {
+ .doOnCompleted( new Action0() {
@Override
public void call() {
timer.stop();
}
- });
+ } );
return o;
}
@@ -300,18 +287,18 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
return Observable.from(entity.getEntity().get());
}
})
- .doOnNext(new Action1<Entity>() {
+ .doOnNext( new Action1<Entity>() {
@Override
- public void call(Entity entity) {
+ public void call( Entity entity ) {
loadMeter.mark();
}
- })
- .doOnCompleted(new Action0() {
+ } )
+ .doOnCompleted( new Action0() {
@Override
public void call() {
timer.stop();
}
- });
+ } );
}
@@ -347,12 +334,12 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
loadMeter.mark();
}
})
- .doOnCompleted(new Action0() {
+ .doOnCompleted( new Action0() {
@Override
public void call() {
timer.stop();
}
- });
+ } );
}
@@ -505,12 +492,12 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
}
}
} )
- .doOnCompleted(new Action0() {
+ .doOnCompleted( new Action0() {
@Override
public void call() {
timer.stop();
}
- });
+ } );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/762e43e1/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTask.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTask.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTask.java
index 27c3db8..55d135b 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTask.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTask.java
@@ -26,10 +26,12 @@ import java.util.UUID;
import com.google.inject.Inject;
import com.google.inject.assistedinject.Assisted;
import org.apache.usergrid.persistence.collection.MvccEntity;
+import org.apache.usergrid.persistence.collection.serialization.MvccEntitySerializationStrategy;
import org.apache.usergrid.persistence.collection.serialization.UniqueValue;
import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
import org.apache.usergrid.persistence.collection.serialization.impl.UniqueValueImpl;
import org.apache.usergrid.persistence.collection.util.EntityUtils;
+import org.apache.usergrid.persistence.core.guice.ProxyImpl;
import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.field.Field;
import org.slf4j.Logger;
@@ -86,7 +88,6 @@ public class EntityVersionCleanupTask implements Task<Void> {
public EntityVersionCleanupTask(
final SerializationFig serializationFig,
final MvccLogEntrySerializationStrategy logEntrySerializationStrategy,
- @ProxyImpl final MvccEntitySerializationStrategy entitySerializationStrategy,
final UniqueValueSerializationStrategy uniqueValueSerializationStrategy,
final Keyspace keyspace,
final Set<EntityVersionDeleted> listeners, // MUST be a set or Guice will not inject
@@ -104,7 +105,7 @@ public class EntityVersionCleanupTask implements Task<Void> {
this.entityId = entityId;
this.version = version;
- includeVersion = includeVersion;
+ this.includeVersion = includeVersion;
}
@@ -146,7 +147,7 @@ public class EntityVersionCleanupTask implements Task<Void> {
.skipWhile( new Func1<UniqueValue, Boolean>() {
@Override
public Boolean call( final UniqueValue uniqueValue ) {
- return version.equals( uniqueValue.getEntityVersion() );
+ return !includeVersion && version.equals( uniqueValue.getEntityVersion() );
}
} )
//buffer our buffer size, then roll them all up in a single batch mutation
@@ -185,7 +186,7 @@ public class EntityVersionCleanupTask implements Task<Void> {
.skipWhile( new Func1<MvccLogEntry, Boolean>() {
@Override
public Boolean call( final MvccLogEntry mvccLogEntry ) {
- return version.equals( mvccLogEntry.getVersion() );
+ return !includeVersion && version.equals( mvccLogEntry.getVersion() );
}
} )
//buffer them for efficiency
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/762e43e1/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV1Impl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV1Impl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV1Impl.java
index 6f8525c..40054ec 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV1Impl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV1Impl.java
@@ -194,7 +194,7 @@ public class MvccEntitySerializationStrategyV1Impl extends MvccEntitySerializati
// it's been deleted, remove it
if ( Arrays.equals( STATE_DELETED, state ) ) {
- return new EntityWrapper( MvccEntity.Status.COMPLETE, Optional.<Entity>absent() );
+ return new EntityWrapper( MvccEntity.Status.DELETED, Optional.<Entity>absent() );
}
Entity storedEntity;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/762e43e1/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3Impl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3Impl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3Impl.java
index 6ad18bb..ace076b 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3Impl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3Impl.java
@@ -310,7 +310,6 @@ public class MvccEntitySerializationStrategyV3Impl implements MvccEntitySerializ
Preconditions.checkNotNull( version, "version is required" );
- TimeUUIDUtils.getMicrosTimeFromUUID(version);
return doWrite( collectionScope, entityId, version, new RowOp() {
@Override
public void doOp( final ColumnListMutation<Boolean> colMutation ) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/762e43e1/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/MvccEntityDataMigrationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/MvccEntityDataMigrationImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/MvccEntityDataMigrationImpl.java
index beaaef9..bd4eafc 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/MvccEntityDataMigrationImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/MvccEntityDataMigrationImpl.java
@@ -30,9 +30,9 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.usergrid.persistence.collection.CollectionScope;
-import org.apache.usergrid.persistence.collection.EntityVersionCleanupFactory;
import org.apache.usergrid.persistence.collection.MvccEntity;
import org.apache.usergrid.persistence.collection.impl.EntityVersionCleanupTask;
+import org.apache.usergrid.persistence.collection.impl.EntityVersionTaskFactory;
import org.apache.usergrid.persistence.collection.serialization.MvccEntitySerializationStrategy;
import org.apache.usergrid.persistence.collection.serialization.UniqueValue;
import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
@@ -75,7 +75,7 @@ public class MvccEntityDataMigrationImpl implements DataMigration<EntityIdScope>
private final Keyspace keyspace;
private final VersionedMigrationSet<MvccEntitySerializationStrategy> allVersions;
private final UniqueValueSerializationStrategy uniqueValueSerializationStrategy;
- private final EntityVersionCleanupFactory entityVersionCleanupFactory;
+ private final EntityVersionTaskFactory entityVersionCleanupFactory;
private final MvccEntitySerializationStrategyV3Impl mvccEntitySerializationStrategyV3;
@@ -84,7 +84,7 @@ public class MvccEntityDataMigrationImpl implements DataMigration<EntityIdScope>
public MvccEntityDataMigrationImpl( final Keyspace keyspace,
final VersionedMigrationSet<MvccEntitySerializationStrategy> allVersions,
final UniqueValueSerializationStrategy uniqueValueSerializationStrategy,
- final EntityVersionCleanupFactory entityVersionCleanupFactory,
+ final EntityVersionTaskFactory entityVersionCleanupFactory,
final MvccEntitySerializationStrategyV3Impl mvccEntitySerializationStrategyV3
) {
@@ -227,7 +227,7 @@ public class MvccEntityDataMigrationImpl implements DataMigration<EntityIdScope>
totalBatch.mergeShallow( mb );
}
- final EntityVersionCleanupTask task = entityVersionCleanupFactory.getTask( message.scope, message.entity.getId(), version );
+ final EntityVersionCleanupTask task = entityVersionCleanupFactory.getCleanupTask( message.scope, message.entity.getId(), version, false );
entityVersionCleanupTasks.add( task );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/762e43e1/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
index c92bb74..f6d9782 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
@@ -23,7 +23,7 @@ import java.util.Arrays;
import java.util.List;
import java.util.UUID;
-import org.apache.usergrid.persistence.collection.mvcc.MvccEntitySerializationStrategy;
+import org.apache.usergrid.persistence.collection.serialization.MvccEntitySerializationStrategy;
import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
import org.apache.usergrid.persistence.collection.serialization.UniqueValueSet;
import org.apache.usergrid.persistence.core.guice.ProxyImpl;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/762e43e1/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTaskTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTaskTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTaskTest.java
index b4921a3..787ab30 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTaskTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTaskTest.java
@@ -128,7 +128,7 @@ public class EntityVersionCleanupTaskTest {
EntityVersionCleanupTask cleanupTask =
new EntityVersionCleanupTask( serializationFig, less, uvss, keyspace, listeners, appScope, entityId,
- version );
+ version, false );
final MutationBatch newBatch = mock( MutationBatch.class );
@@ -209,8 +209,7 @@ public class EntityVersionCleanupTaskTest {
EntityVersionCleanupTask cleanupTask =
- new EntityVersionCleanupTask( serializationFig, less, uvss, keyspace, listeners, appScope, entityId,
- version );
+ new EntityVersionCleanupTask( serializationFig, less, uvss, keyspace, listeners, appScope, entityId, version, false );
final MutationBatch newBatch = mock( MutationBatch.class );
@@ -290,7 +289,7 @@ public class EntityVersionCleanupTaskTest {
EntityVersionCleanupTask cleanupTask =
new EntityVersionCleanupTask( serializationFig, less, uvss, keyspace, listeners, appScope, entityId,
- version );
+ version, false );
final MutationBatch newBatch = mock( MutationBatch.class );
@@ -390,7 +389,7 @@ public class EntityVersionCleanupTaskTest {
EntityVersionCleanupTask cleanupTask =
new EntityVersionCleanupTask( serializationFig, less, uvss, keyspace, listeners, appScope, entityId,
- version );
+ version, false );
final MutationBatch newBatch = mock( MutationBatch.class );
@@ -517,7 +516,7 @@ public class EntityVersionCleanupTaskTest {
EntityVersionCleanupTask cleanupTask =
new EntityVersionCleanupTask( serializationFig, less, uvss, keyspace, listeners, appScope, entityId,
- version );
+ version, false);
final MutationBatch newBatch = mock( MutationBatch.class );
@@ -645,7 +644,7 @@ public class EntityVersionCleanupTaskTest {
EntityVersionCleanupTask cleanupTask =
new EntityVersionCleanupTask( serializationFig, less, uvss, keyspace, listeners, appScope, entityId,
- version );
+ version, false );
final MutationBatch newBatch = mock( MutationBatch.class );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/762e43e1/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV1_3Test.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV1_3Test.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV1_3Test.java
index 42b10c4..56abfd8 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV1_3Test.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV1_3Test.java
@@ -34,9 +34,12 @@ import org.apache.usergrid.persistence.core.test.UseModules;
import com.google.inject.Inject;
+import net.jcip.annotations.NotThreadSafe;
+
@RunWith( ITRunner.class )
@UseModules( TestCollectionModule.class )
+@NotThreadSafe//anything that changes the system version state is not safe to be run concurrently
public class MvccEntitySerializationStrategyProxyV1_3Test extends MvccEntitySerializationStrategyImplTest {
@Inject
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/762e43e1/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV2_3Test.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV2_3Test.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV2_3Test.java
index da496f7..b372fbf 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV2_3Test.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV2_3Test.java
@@ -34,9 +34,12 @@ import org.apache.usergrid.persistence.core.test.UseModules;
import com.google.inject.Inject;
+import net.jcip.annotations.NotThreadSafe;
+
@RunWith( ITRunner.class )
@UseModules( TestCollectionModule.class )
+@NotThreadSafe//anything that changes the system version state is not safe to be run concurrently
public class MvccEntitySerializationStrategyProxyV2_3Test extends MvccEntitySerializationStrategyV2Test {
@Inject
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/762e43e1/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV1ImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV1ImplTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV1ImplTest.java
index 4e0c8cc..b0dba5d 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV1ImplTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV1ImplTest.java
@@ -49,6 +49,8 @@ import com.google.common.base.Optional;
import com.google.inject.Inject;
import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
+import net.jcip.annotations.NotThreadSafe;
+
@RunWith( ITRunner.class )
@UseModules( TestCollectionModule.class )
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/762e43e1/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/GraphManagerFactoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/GraphManagerFactoryImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/GraphManagerFactoryImpl.java
index 732f0bf..7c88c33 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/GraphManagerFactoryImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/GraphManagerFactoryImpl.java
@@ -26,6 +26,8 @@ import com.google.common.cache.LoadingCache;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import com.google.inject.assistedinject.Assisted;
+
+import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.graph.GraphFig;
import org.apache.usergrid.persistence.graph.GraphManager;
@@ -51,29 +53,28 @@ public class GraphManagerFactoryImpl implements GraphManagerFactory {
private final GraphFig graphFig;
private final EdgeDeleteListener edgeDeleteListener;
private final NodeDeleteListener nodeDeleteListener;
+ private final MetricsFactory metricsFactory;
private LoadingCache<ApplicationScope, GraphManager> gmCache =
CacheBuilder.newBuilder().maximumSize( 1000 ).build( new CacheLoader<ApplicationScope, GraphManager>() {
public GraphManager load(
ApplicationScope scope ) {
- return new GraphManagerImpl(edgeMetadataSerialization,edgeSerialization,nodeSerialization,graphFig,edgeDeleteListener,nodeDeleteListener,scope);
+ return new GraphManagerImpl(edgeMetadataSerialization,edgeSerialization,nodeSerialization,graphFig,edgeDeleteListener,nodeDeleteListener,scope, metricsFactory);
}
} );
@Inject
- public GraphManagerFactoryImpl(final EdgeMetadataSerialization edgeMetadataSerialization,
- final EdgeSerialization edgeSerialization,
- final NodeSerialization nodeSerialization,
- final GraphFig graphFig,
- final EdgeDeleteListener edgeDeleteListener,
- final NodeDeleteListener nodeDeleteListener
- ){
+ public GraphManagerFactoryImpl( final EdgeMetadataSerialization edgeMetadataSerialization, final
+ EdgeSerialization edgeSerialization,
+ final NodeSerialization nodeSerialization, final GraphFig graphFig, final EdgeDeleteListener edgeDeleteListener,
+ final NodeDeleteListener nodeDeleteListener, final MetricsFactory metricsFactory ){
this.edgeMetadataSerialization = edgeMetadataSerialization;
this.edgeSerialization = edgeSerialization;
this.nodeSerialization = nodeSerialization;
this.graphFig = graphFig;
this.edgeDeleteListener = edgeDeleteListener;
this.nodeDeleteListener = nodeDeleteListener;
+ this.metricsFactory = metricsFactory;
}
@Override
[38/50] [abbrv] incubator-usergrid git commit: Revert "fix for list
types changing from string to int or whatever"
Posted by sf...@apache.org.
Revert "fix for list types changing from string to int or whatever"
This reverts commit e5cab76f8b8fc1e4a60ea23274e7a8a0cc79f759.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/b09dc435
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/b09dc435
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/b09dc435
Branch: refs/heads/USERGRID-480
Commit: b09dc43526cd27dcf75fa09dc5c40a288ca30225
Parents: 762e43e
Author: Todd Nine <tn...@apigee.com>
Authored: Tue Mar 17 17:30:27 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Tue Mar 17 17:30:27 2015 -0600
----------------------------------------------------------------------
.../index/impl/EsEntityIndexBatchImpl.java | 14 ++-
.../persistence/index/impl/IndexingUtils.java | 109 +++++++++----------
.../persistence/index/impl/EntityIndexTest.java | 38 +------
3 files changed, 70 insertions(+), 91 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b09dc435/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
index 4c32cc1..92312d2 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
@@ -56,7 +56,15 @@ import com.codahale.metrics.Timer;
import rx.Observable;
import rx.functions.Func1;
-import static org.apache.usergrid.persistence.index.impl.IndexingUtils.*;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ANALYZED_STRING_PREFIX;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.BOOLEAN_PREFIX;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ENTITYID_ID_FIELDNAME;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ENTITY_CONTEXT_FIELDNAME;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.GEO_PREFIX;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.NUMBER_PREFIX;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.STRING_PREFIX;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.createContextName;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.createIndexDocId;
public class EsEntityIndexBatchImpl implements EntityIndexBatch {
@@ -235,7 +243,7 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
if ( f instanceof ListField ) {
List list = ( List ) field.getValue();
- entityMap.put(LIST_PREFIX + field.getName().toLowerCase(),
+ entityMap.put( field.getName().toLowerCase(),
new ArrayList( processCollectionForMap( list ) ) );
if ( !list.isEmpty() ) {
@@ -247,7 +255,7 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
}
else if ( f instanceof ArrayField ) {
List list = ( List ) field.getValue();
- entityMap.put(ARRAY_PREFIX + field.getName().toLowerCase(),
+ entityMap.put( field.getName().toLowerCase(),
new ArrayList( processCollectionForMap( list ) ) );
}
else if ( f instanceof SetField ) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b09dc435/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
index 41a4680..ffd98e9 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
@@ -34,9 +34,6 @@ public class IndexingUtils {
public static final String STRING_PREFIX = "su_";
public static final String ANALYZED_STRING_PREFIX = "sa_";
- public static final String ARRAY_PREFIX = "ar_";
- public static final String LIST_PREFIX = "li_";
-
public static final String GEO_PREFIX = "go_";
public static final String NUMBER_PREFIX = "nu_";
public static final String BOOLEAN_PREFIX = "bu_";
@@ -156,75 +153,75 @@ public class IndexingUtils {
/** add routing "_routing":{ "required":false, "path":"ug_entityId" **/
.startObject("_routing").field("required",true).field("path",ENTITYID_ID_FIELDNAME).endObject()
- .startArray("dynamic_templates")
+ .startArray("dynamic_templates")
// we need most specific mappings first since it's a stop on match algorithm
- .startObject()
+ .startObject()
- .startObject("entity_id_template" )
- .field("match", IndexingUtils.ENTITYID_ID_FIELDNAME )
- .field("match_mapping_type", "string" )
- .startObject( "mapping").field("type", "string" )
- .field("index", "not_analyzed" )
- .endObject()
- .endObject()
- .endObject()
+ .startObject( "entity_id_template" )
+ .field( "match", IndexingUtils.ENTITYID_ID_FIELDNAME )
+ .field( "match_mapping_type", "string" )
+ .startObject( "mapping" ).field( "type", "string" )
+ .field( "index", "not_analyzed" )
+ .endObject()
+ .endObject()
+ .endObject()
- .startObject()
- .startObject("entity_context_template" )
+ .startObject()
+ .startObject( "entity_context_template" )
.field( "match", IndexingUtils.ENTITY_CONTEXT_FIELDNAME )
- .field("match_mapping_type", "string" )
- .startObject( "mapping" )
+ .field( "match_mapping_type", "string" )
+ .startObject( "mapping" )
.field( "type", "string" )
- .field("index", "not_analyzed").endObject()
- .endObject()
- .endObject()
-
- .startObject()
- .startObject("entity_version_template" )
- .field("match", IndexingUtils.ENTITY_VERSION_FIELDNAME )
- .field("match_mapping_type", "string" )
- .startObject( "mapping").field("type", "long" )
- .endObject()
- .endObject()
- .endObject()
+ .field( "index", "not_analyzed" ).endObject()
+ .endObject()
+ .endObject()
+
+ .startObject()
+ .startObject( "entity_version_template" )
+ .field( "match", IndexingUtils.ENTITY_VERSION_FIELDNAME )
+ .field( "match_mapping_type", "string" )
+ .startObject( "mapping" ).field( "type", "long" )
+ .endObject()
+ .endObject()
+ .endObject()
// any string with field name that starts with sa_ gets analyzed
- .startObject()
- .startObject("template_1" )
+ .startObject()
+ .startObject( "template_1" )
.field( "match", ANALYZED_STRING_PREFIX + "*" )
- .field("match_mapping_type", "string" ).startObject( "mapping" )
+ .field( "match_mapping_type", "string" ).startObject( "mapping" )
.field( "type", "string" )
- .field("index", "analyzed" )
- .endObject()
- .endObject()
+ .field( "index", "analyzed" )
+ .endObject()
+ .endObject()
- .endObject()
+ .endObject()
// all other strings are not analyzed
- .startObject()
- .startObject("template_2" )
+ .startObject()
+ .startObject( "template_2" )
//todo, should be string prefix, remove 2 field mapping
- .field("match", "*" )
- .field("match_mapping_type", "string" )
- .startObject( "mapping" )
- .field("type", "string" )
- .field("index", "not_analyzed" )
- .endObject()
- .endObject()
- .endObject()
+ .field( "match", "*" )
+ .field( "match_mapping_type", "string" )
+ .startObject( "mapping" )
+ .field( "type", "string" )
+ .field( "index", "not_analyzed" )
+ .endObject()
+ .endObject()
+ .endObject()
// fields names starting with go_ get geo-indexed
- .startObject()
- .startObject("template_3" )
- .field("match", GEO_PREFIX + "location" )
- .startObject( "mapping" )
- .field("type", "geo_point" )
- .endObject()
- .endObject()
- .endObject()
-
- .endArray()
+ .startObject()
+ .startObject( "template_3" )
+ .field( "match", GEO_PREFIX + "location" )
+ .startObject( "mapping" )
+ .field( "type", "geo_point" )
+ .endObject()
+ .endObject()
+ .endObject()
+
+ .endArray()
.endObject();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b09dc435/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
index 9cb5297..ca9bf79 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
@@ -26,7 +26,9 @@ import java.util.concurrent.atomic.AtomicLong;
import org.apache.usergrid.persistence.core.guice.MigrationManagerRule;
import org.apache.usergrid.persistence.index.*;
-import org.apache.usergrid.persistence.model.field.*;
+import org.apache.usergrid.persistence.model.field.ArrayField;
+import org.apache.usergrid.persistence.model.field.EntityObjectField;
+import org.apache.usergrid.persistence.model.field.UUIDField;
import org.apache.usergrid.persistence.model.field.value.EntityObject;
import org.junit.Ignore;
import org.junit.Rule;
@@ -49,6 +51,7 @@ import org.apache.usergrid.persistence.index.utils.UUIDUtils;
import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.entity.SimpleId;
+import org.apache.usergrid.persistence.model.field.StringField;
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
import com.fasterxml.jackson.core.type.TypeReference;
@@ -96,7 +99,7 @@ public class EntityIndexTest extends BaseIT {
}
@Test
-// @Ignore("this is a problem i will work on when i can breathe")
+ @Ignore("this is a problem i will work on when i can breathe")
public void testIndexVariations() throws IOException {
Id appId = new SimpleId( "application" );
@@ -116,8 +119,6 @@ public class EntityIndexTest extends BaseIT {
batch.index(indexScope, entity);
batch.execute().get();
- entity = new Entity( entityType );
- entity.setField(new UUIDField(IndexingUtils.ENTITYID_ID_FIELDNAME, UUID.randomUUID()));
EntityUtils.setVersion(entity, UUIDGenerator.newTimeUUID());
List<String> list = new ArrayList<>();
list.add("test");
@@ -125,9 +126,6 @@ public class EntityIndexTest extends BaseIT {
batch.index(indexScope, entity);
batch.execute().get();
-
- entity = new Entity( entityType );
- entity.setField(new UUIDField(IndexingUtils.ENTITYID_ID_FIELDNAME, UUID.randomUUID()));
EntityUtils.setVersion(entity, UUIDGenerator.newTimeUUID());
EntityObject testObj = new EntityObject();
testObj.setField(new StringField("test","testFiedl"));
@@ -135,32 +133,9 @@ public class EntityIndexTest extends BaseIT {
batch.index(indexScope, entity);
batch.execute().get();
- entity = new Entity( entityType );
- entity.setField(new UUIDField(IndexingUtils.ENTITYID_ID_FIELDNAME, UUID.randomUUID()));
- EntityUtils.setVersion(entity, UUIDGenerator.newTimeUUID());
- List<Integer> listint = new ArrayList<>();
- listint.add(0);
- entity.setField(new ArrayField<Integer>("testfield", listint));
- batch.index(indexScope, entity);
- batch.execute().get();
-
- entity = new Entity( entityType );
- entity.setField(new UUIDField(IndexingUtils.ENTITYID_ID_FIELDNAME, UUID.randomUUID()));
- EntityUtils.setVersion(entity, UUIDGenerator.newTimeUUID());
- List<EntityObject> listObj = new ArrayList<>();
- EntityObject listObjField = new EntityObject();
- listObjField.setField(new StringField("testasf","somevalue"));
- listObj.add(listObjField);
- listObjField = new EntityObject();
- listObjField.setField(new IntegerField("testasf",0));
- listObj.add(listObjField);
- entity.setField(new ArrayField<EntityObject>("testfield", listObj));
- batch.index(indexScope, entity);
- batch.execute().get();
-
entityIndex.refresh();
- testQuery(indexScope, searchTypes, entityIndex, "select *", 5);
+ testQueries( indexScope, searchTypes, entityIndex );
}
@Test
@@ -288,7 +263,6 @@ public class EntityIndexTest extends BaseIT {
entityIndex.refresh();
//Hilda Youn
-
testQuery(indexScope, searchTypes, entityIndex, "name = 'Bowers Oneil'", 0);
}
[43/50] [abbrv] incubator-usergrid git commit: Removed exception
swallowing
Posted by sf...@apache.org.
Removed exception swallowing
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/61c5f9cc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/61c5f9cc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/61c5f9cc
Branch: refs/heads/USERGRID-480
Commit: 61c5f9ccadf74dc997235d5d9a71403e6cd7f60c
Parents: d8f949b
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Mar 18 17:45:43 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Mar 18 17:45:43 2015 -0600
----------------------------------------------------------------------
.../organizations/applications/ApplicationsResource.java | 6 ------
1 file changed, 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/61c5f9cc/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/ApplicationsResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/ApplicationsResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/ApplicationsResource.java
index 2a5ce9d..f94fa34 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/ApplicationsResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/ApplicationsResource.java
@@ -107,8 +107,6 @@ public class ApplicationsResource extends AbstractContextResource {
logger.debug("newApplicationForOrganizationFromForm");
- try {
-
Preconditions.checkArgument( !isEmpty( applicationName ),
"The 'name' parameter is required and cannot be empty: " + applicationName );
@@ -123,10 +121,6 @@ public class ApplicationsResource extends AbstractContextResource {
response.setResults( management.getApplicationMetadata( applicationInfo.getId() ) );
return new JSONWithPadding( response, callback );
- } catch (Exception e) {
- logger.debug("Error in newApplicationForOrganizationFromForm", e);
- return null;
- }
}
[08/50] [abbrv] incubator-usergrid git commit: Updated graph migration
Posted by sf...@apache.org.
Updated graph migration
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/fa69be86
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/fa69be86
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/fa69be86
Branch: refs/heads/USERGRID-480
Commit: fa69be86c48e916a7a368ca3bef6101bc0e81671
Parents: a55c784
Author: Todd Nine <tn...@apigee.com>
Authored: Mon Mar 2 18:43:13 2015 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Mon Mar 2 18:43:13 2015 -0700
----------------------------------------------------------------------
.../impl/CollectionDataVersions.java | 6 +-
...vccEntitySerializationStrategyProxyImpl.java | 2 +-
.../MvccEntitySerializationStrategyV1Impl.java | 3 +-
.../MvccEntitySerializationStrategyV2Impl.java | 2 +-
.../MvccEntitySerializationStrategyV3Impl.java | 6 +-
.../serialization/impl/SerializationModule.java | 61 +++----
.../migration/CollectionMigrationPlugin.java | 83 +---------
.../migration/MvccEntityDataMigrationImpl.java | 3 -
.../data/newimpls/AbstractMigrationPlugin.java | 125 +++++++++++++++
.../persistence/graph/guice/GraphModule.java | 82 ++++++++--
.../EdgeMetadataSerialization.java | 3 +-
.../serialization/EdgeMigrationStrategy.java | 33 ----
.../impl/EdgeDataMigrationImpl.java | 52 +++---
.../EdgeMetadataSerializationProxyImpl.java | 158 +++++++++++--------
.../impl/EdgeMetadataSerializationV1Impl.java | 6 +
.../impl/EdgeMetadataSerializationV2Impl.java | 6 +
.../serialization/impl/GraphDataVersions.java | 43 +++++
.../impl/migration/GraphMigrationPlugin.java | 63 ++++++++
18 files changed, 464 insertions(+), 273 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fa69be86/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/CollectionDataVersions.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/CollectionDataVersions.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/CollectionDataVersions.java
index 6e8be45..76d35d7 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/CollectionDataVersions.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/CollectionDataVersions.java
@@ -28,9 +28,9 @@ package org.apache.usergrid.persistence.collection.serialization.impl;
* Versions of data as they exist across our system
*/
public enum CollectionDataVersions{
- ZERO(0),
- ONE(1),
- TWO(2);
+ INITIAL(0),
+ BUFFER_SHORT_FIX(1),
+ LOG_REMOVAL(2);
private final int version;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fa69be86/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyImpl.java
index 02dfd9d..19bfc9e 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyImpl.java
@@ -187,7 +187,7 @@ public class MvccEntitySerializationStrategyProxyImpl implements MvccEntitySeria
@Override
public int getImplementationVersion() {
- return 0;
+ throw new UnsupportedOperationException("Not supported in the proxy");
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fa69be86/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV1Impl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV1Impl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV1Impl.java
index 1dab673..6f8525c 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV1Impl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV1Impl.java
@@ -26,7 +26,6 @@ import java.util.UUID;
import org.apache.usergrid.persistence.collection.MvccEntity;
import org.apache.usergrid.persistence.collection.exception.DataCorruptionException;
import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
-import org.apache.usergrid.persistence.core.astyanax.CassandraConfig;
import org.apache.usergrid.persistence.core.astyanax.CassandraFig;
import org.apache.usergrid.persistence.core.astyanax.IdRowCompositeSerializer;
import org.apache.usergrid.persistence.core.astyanax.MultiTennantColumnFamily;
@@ -90,7 +89,7 @@ public class MvccEntitySerializationStrategyV1Impl extends MvccEntitySerializati
@Override
public int getImplementationVersion() {
- return CollectionDataVersions.ZERO.getVersion();
+ return CollectionDataVersions.INITIAL.getVersion();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fa69be86/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2Impl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2Impl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2Impl.java
index 1f65fcb..ed97e80 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2Impl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2Impl.java
@@ -93,7 +93,7 @@ public class MvccEntitySerializationStrategyV2Impl extends MvccEntitySerializati
@Override
public int getImplementationVersion() {
- return CollectionDataVersions.ONE.getVersion();
+ return CollectionDataVersions.BUFFER_SHORT_FIX.getVersion();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fa69be86/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3Impl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3Impl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3Impl.java
index 7b8aac1..ba58ee2 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3Impl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3Impl.java
@@ -14,8 +14,6 @@ import org.slf4j.LoggerFactory;
import org.apache.cassandra.db.marshal.BooleanType;
import org.apache.cassandra.db.marshal.BytesType;
-import org.apache.cassandra.db.marshal.ReversedType;
-import org.apache.cassandra.db.marshal.UUIDType;
import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.collection.EntitySet;
@@ -44,8 +42,6 @@ import org.apache.usergrid.persistence.model.util.UUIDGenerator;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.dataformat.smile.SmileFactory;
-import com.fasterxml.uuid.UUIDComparator;
-import com.fasterxml.uuid.impl.UUIDUtil;
import com.google.common.base.Optional;
import com.google.common.base.Preconditions;
import com.google.inject.Inject;
@@ -383,7 +379,7 @@ public class MvccEntitySerializationStrategyV3Impl implements MvccEntitySerializ
@Override
public int getImplementationVersion() {
- return CollectionDataVersions.TWO.getVersion();
+ return CollectionDataVersions.LOG_REMOVAL.getVersion();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fa69be86/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/SerializationModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/SerializationModule.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/SerializationModule.java
index f32f4f9..241a274 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/SerializationModule.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/SerializationModule.java
@@ -35,6 +35,7 @@ import com.google.inject.AbstractModule;
import com.google.inject.Inject;
import com.google.inject.Key;
import com.google.inject.Provider;
+import com.google.inject.Provides;
import com.google.inject.Singleton;
import com.google.inject.TypeLiteral;
import com.google.inject.multibindings.Multibinder;
@@ -64,10 +65,6 @@ public class SerializationModule extends AbstractModule {
bind( MvccEntitySerializationStrategyV3Impl.class );
- bind( new TypeLiteral<VersionedMigrationSet<MvccEntitySerializationStrategy>>() {} )
- .toProvider( MvccEntitySerializationStrategyProvider.class );
-
-
//migrations
//we want to make sure our generics are retained, so we use a typeliteral
Multibinder<DataMigration2<EntityIdScope>> dataMigrationMultibinder =
@@ -99,50 +96,36 @@ public class SerializationModule extends AbstractModule {
bind( SettingsValidation.class ).asEagerSingleton();
}
+ /**
+ * Configure via explicit declaration the migration path we can follow
+ * @param v1
+ * @param v2
+ * @param v3
+ * @return
+ */
+ @Singleton
+ @Inject
+ @Provides
+ public VersionedMigrationSet<MvccEntitySerializationStrategy> getVersions(final MvccEntitySerializationStrategyV1Impl v1, final MvccEntitySerializationStrategyV2Impl v2, final MvccEntitySerializationStrategyV3Impl v3){
- @Singleton
- public static final class MvccEntitySerializationStrategyProvider
- implements Provider<VersionedMigrationSet<MvccEntitySerializationStrategy>> {
-
-
- private final MvccEntitySerializationStrategyV1Impl v1;
- private final MvccEntitySerializationStrategyV2Impl v2;
- private final MvccEntitySerializationStrategyV3Impl v3;
+ //we must perform a migration from v1 to v3 in order to maintain consistency
+ MigrationRelationship<MvccEntitySerializationStrategy> v1Tov3 = new MigrationRelationship<>( v1, v3 );
- @Inject
- public MvccEntitySerializationStrategyProvider( final MvccEntitySerializationStrategyV1Impl v1,
- final MvccEntitySerializationStrategyV2Impl v2,
- final MvccEntitySerializationStrategyV3Impl v3 ) {
- this.v1 = v1;
- this.v2 = v2;
- this.v3 = v3;
- }
+ //we must migrate from 2 to 3, this is a bridge that must happen to maintain data consistency
+ MigrationRelationship<MvccEntitySerializationStrategy> v2Tov3 = new MigrationRelationship<>( v2, v3 );
- @Override
- public VersionedMigrationSet<MvccEntitySerializationStrategy> get() {
- //we must perform a migration from v1 to v3 in order to maintain consistency
- MigrationRelationship<MvccEntitySerializationStrategy> v1Tov3 = new MigrationRelationship<>( v1, v3 );
+ //note that we MUST migrate to v3 before our next migration, if v4 and v5 is implemented we will need a v3->v5 and a v4->v5 set
+ MigrationRelationship<MvccEntitySerializationStrategy> current = new MigrationRelationship<MvccEntitySerializationStrategy>( v3, v3 );
- //we must migrate from 2 to 3, this is a bridge that must happen to maintain data consistency
- MigrationRelationship<MvccEntitySerializationStrategy> v2Tov3 = new MigrationRelationship<>( v2, v3 );
+ //now create our set of versions
+ VersionedMigrationSet<MvccEntitySerializationStrategy> set = new VersionedMigrationSet<>( v1Tov3, v2Tov3, current );
+ return set;
- //note that we MUST migrate to v3 before our next migration, if v4 and v5 is implemented we will need a
- // v3->v5 and a v4->v5 set
- MigrationRelationship<MvccEntitySerializationStrategy> current =
- new MigrationRelationship<MvccEntitySerializationStrategy>( v3, v3 );
+ }
-
- //now create our set of versions
- VersionedMigrationSet<MvccEntitySerializationStrategy> set =
- new VersionedMigrationSet<>( v1Tov3, v2Tov3, current );
-
-
- return set;
- }
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fa69be86/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/CollectionMigrationPlugin.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/CollectionMigrationPlugin.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/CollectionMigrationPlugin.java
index d0663c2..0cf25b2 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/CollectionMigrationPlugin.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/CollectionMigrationPlugin.java
@@ -26,15 +26,10 @@ package org.apache.usergrid.persistence.collection.serialization.impl.migration;
import java.util.Set;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.usergrid.persistence.core.migration.data.DataMigrationException;
import org.apache.usergrid.persistence.core.migration.data.MigrationInfoSerialization;
+import org.apache.usergrid.persistence.core.migration.data.newimpls.AbstractMigrationPlugin;
import org.apache.usergrid.persistence.core.migration.data.newimpls.DataMigration2;
import org.apache.usergrid.persistence.core.migration.data.newimpls.MigrationDataProvider;
-import org.apache.usergrid.persistence.core.migration.data.newimpls.MigrationPlugin;
-import org.apache.usergrid.persistence.core.migration.data.newimpls.ProgressObserver;
import com.google.inject.Inject;
import com.google.inject.Singleton;
@@ -44,25 +39,17 @@ import com.google.inject.Singleton;
* Migration plugin for the collection module
*/
@Singleton
-public class CollectionMigrationPlugin implements MigrationPlugin {
-
-
- private static final Logger LOG = LoggerFactory.getLogger( CollectionMigrationPlugin.class );
+public class CollectionMigrationPlugin extends AbstractMigrationPlugin<EntityIdScope> {
public static final String PLUGIN_NAME = "collections-entity-data";
- private final Set<DataMigration2<EntityIdScope>> entityDataMigrations;
- private final MigrationDataProvider<EntityIdScope> entityIdScopeDataMigrationProvider;
- private final MigrationInfoSerialization migrationInfoSerialization;
@Inject
public CollectionMigrationPlugin( final Set<DataMigration2<EntityIdScope>> entityDataMigrations,
final MigrationDataProvider<EntityIdScope> entityIdScopeDataMigrationProvider,
final MigrationInfoSerialization migrationInfoSerialization ) {
- this.entityDataMigrations = entityDataMigrations;
- this.entityIdScopeDataMigrationProvider = entityIdScopeDataMigrationProvider;
- this.migrationInfoSerialization = migrationInfoSerialization;
+ super( entityDataMigrations, entityIdScopeDataMigrationProvider, migrationInfoSerialization );
}
@@ -72,68 +59,4 @@ public class CollectionMigrationPlugin implements MigrationPlugin {
}
- @Override
- public void run( final ProgressObserver observer ) {
-
- //run until complete
- while(runMigration( observer )){
- LOG.info( "Migration complete, checking for next run" );
- }
-
- }
-
-
- @Override
- public int getMaxVersion() {
-
- int max = 0;
-
- for(DataMigration2<EntityIdScope> entityMigration: entityDataMigrations){
- max = Math.max( max, entityMigration.getMaxVersion() );
- }
-
- return max;
- }
-
-
- /**
- * Try to run the migration
- *
- * @return True if we ran a migration
- */
- private boolean runMigration( final ProgressObserver po ) {
- DataMigration2<EntityIdScope> migrationToExecute = null;
-
-
- final int version = migrationInfoSerialization.getVersion( PLUGIN_NAME );
-
- for ( DataMigration2<EntityIdScope> entityMigration : entityDataMigrations ) {
- if ( entityMigration.supports( version ) ) {
- if ( migrationToExecute != null ) {
- throw new DataMigrationException(
- "Two migrations attempted to migration the same version, this is not allowed. Class '"
- + migrationToExecute.getClass().getName() + "' and class '" + entityMigration
- .getClass().getName()
- + "' both support this version. This means something is wired incorrectly" );
- }
-
- migrationToExecute = entityMigration;
- }
- }
-
- if(migrationToExecute == null){
- LOG.info( "No migrations found to execute" );
- return false;
- }
-
- //run the migration
- final int newSystemVersion = migrationToExecute.migrate( version, entityIdScopeDataMigrationProvider, po );
-
- migrationInfoSerialization.setVersion( PLUGIN_NAME, newSystemVersion );
-
- //signal we've run a migration and return
- return true;
-
-
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fa69be86/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/MvccEntityDataMigrationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/MvccEntityDataMigrationImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/MvccEntityDataMigrationImpl.java
index 48e0195..108a4d8 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/MvccEntityDataMigrationImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/MvccEntityDataMigrationImpl.java
@@ -73,7 +73,6 @@ public class MvccEntityDataMigrationImpl implements DataMigration2<EntityIdScope
private final Keyspace keyspace;
private final VersionedMigrationSet<MvccEntitySerializationStrategy> allVersions;
private final UniqueValueSerializationStrategy uniqueValueSerializationStrategy;
- private final MigrationInfoSerialization migrationInfoSerialization;
private final EntityVersionCleanupFactory entityVersionCleanupFactory;
private final MvccEntitySerializationStrategyV3Impl mvccEntitySerializationStrategyV3;
@@ -82,14 +81,12 @@ public class MvccEntityDataMigrationImpl implements DataMigration2<EntityIdScope
public MvccEntityDataMigrationImpl( final Keyspace keyspace,
final VersionedMigrationSet<MvccEntitySerializationStrategy> allVersions,
final UniqueValueSerializationStrategy uniqueValueSerializationStrategy,
- final MigrationInfoSerialization migrationInfoSerialization,
final EntityVersionCleanupFactory entityVersionCleanupFactory,
final MvccEntitySerializationStrategyV3Impl mvccEntitySerializationStrategyV3 ) {
this.keyspace = keyspace;
this.allVersions = allVersions;
this.uniqueValueSerializationStrategy = uniqueValueSerializationStrategy;
- this.migrationInfoSerialization = migrationInfoSerialization;
this.entityVersionCleanupFactory = entityVersionCleanupFactory;
this.mvccEntitySerializationStrategyV3 = mvccEntitySerializationStrategyV3;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fa69be86/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/AbstractMigrationPlugin.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/AbstractMigrationPlugin.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/AbstractMigrationPlugin.java
new file mode 100644
index 0000000..7b3aa00
--- /dev/null
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/AbstractMigrationPlugin.java
@@ -0,0 +1,125 @@
+/*
+ *
+ * *
+ * * Licensed to the Apache Software Foundation (ASF) under one
+ * * or more contributor license agreements. See the NOTICE file
+ * * distributed with this work for additional information
+ * * regarding copyright ownership. The ASF licenses this file
+ * * to you under the Apache License, Version 2.0 (the
+ * * "License"); you may not use this file except in compliance
+ * * with the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing,
+ * * software distributed under the License is distributed on an
+ * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * * KIND, either express or implied. See the License for the
+ * * specific language governing permissions and limitations
+ * * under the License.
+ * *
+ *
+ */
+
+package org.apache.usergrid.persistence.core.migration.data.newimpls;
+
+
+import java.util.Set;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.usergrid.persistence.core.migration.data.DataMigrationException;
+import org.apache.usergrid.persistence.core.migration.data.MigrationInfoSerialization;
+
+
+/**
+ * Standard implementation logic for plugins to extend
+ * @param <T>
+ */
+public abstract class AbstractMigrationPlugin<T> implements MigrationPlugin {
+
+
+
+ private static final Logger LOG = LoggerFactory.getLogger( AbstractMigrationPlugin.class );
+
+
+ private final Set<DataMigration2<T>> entityDataMigrations;
+ private final MigrationDataProvider<T> entityIdScopeDataMigrationProvider;
+ private final MigrationInfoSerialization migrationInfoSerialization;
+
+
+ protected AbstractMigrationPlugin( final Set<DataMigration2<T>> entityDataMigrations,
+ final MigrationDataProvider<T> entityIdScopeDataMigrationProvider,
+ final MigrationInfoSerialization migrationInfoSerialization ) {
+ this.entityDataMigrations = entityDataMigrations;
+ this.entityIdScopeDataMigrationProvider = entityIdScopeDataMigrationProvider;
+ this.migrationInfoSerialization = migrationInfoSerialization;
+ }
+
+
+ @Override
+ public void run( final ProgressObserver observer ) {
+
+ //run until complete
+ while(runMigration( observer )){
+ LOG.info( "Migration complete, checking for next run" );
+ }
+
+ }
+
+
+ @Override
+ public int getMaxVersion() {
+
+ int max = 0;
+
+ for(DataMigration2<T> entityMigration: entityDataMigrations){
+ max = Math.max( max, entityMigration.getMaxVersion() );
+ }
+
+ return max;
+ }
+
+
+ /**
+ * Try to run the migration
+ *
+ * @return True if we ran a migration
+ */
+ private boolean runMigration( final ProgressObserver po ) {
+ DataMigration2<T> migrationToExecute = null;
+
+
+ final int version = migrationInfoSerialization.getVersion( getName() );
+
+ for ( DataMigration2<T> entityMigration : entityDataMigrations ) {
+ if ( entityMigration.supports( version ) ) {
+ if ( migrationToExecute != null ) {
+ throw new DataMigrationException(
+ "Two migrations attempted to migration the same version, this is not allowed. Class '"
+ + migrationToExecute.getClass().getName() + "' and class '" + entityMigration
+ .getClass().getName()
+ + "' both support this version. This means something is wired incorrectly" );
+ }
+
+ migrationToExecute = entityMigration;
+ }
+ }
+
+ if(migrationToExecute == null){
+ LOG.info( "No migrations found to execute" );
+ return false;
+ }
+
+ //run the migration
+ final int newSystemVersion = migrationToExecute.migrate( version, entityIdScopeDataMigrationProvider, po );
+
+ migrationInfoSerialization.setVersion( getName(), newSystemVersion );
+
+ //signal we've run a migration and return
+ return true;
+
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fa69be86/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/guice/GraphModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/guice/GraphModule.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/guice/GraphModule.java
index d513b05..84fbaac 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/guice/GraphModule.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/guice/GraphModule.java
@@ -19,18 +19,17 @@
package org.apache.usergrid.persistence.graph.guice;
-import org.apache.usergrid.persistence.core.guice.V1Impl;
-import org.apache.usergrid.persistence.core.guice.V2Impl;
-import org.apache.usergrid.persistence.core.migration.data.ApplicationDataMigration;
-import org.apache.usergrid.persistence.core.migration.data.DataMigration;
-import org.apache.usergrid.persistence.graph.serialization.*;
-import org.apache.usergrid.persistence.graph.serialization.impl.*;
import org.safehaus.guicyfig.GuicyFigModule;
import org.apache.usergrid.persistence.core.consistency.TimeService;
import org.apache.usergrid.persistence.core.consistency.TimeServiceImpl;
import org.apache.usergrid.persistence.core.guice.ProxyImpl;
+import org.apache.usergrid.persistence.core.migration.data.newimpls.DataMigration2;
+import org.apache.usergrid.persistence.core.migration.data.newimpls.MigrationPlugin;
+import org.apache.usergrid.persistence.core.migration.data.newimpls.MigrationRelationship;
+import org.apache.usergrid.persistence.core.migration.data.newimpls.VersionedMigrationSet;
import org.apache.usergrid.persistence.core.migration.schema.Migration;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.task.NamedTaskExecutorImpl;
import org.apache.usergrid.persistence.core.task.TaskExecutor;
import org.apache.usergrid.persistence.graph.GraphFig;
@@ -43,6 +42,21 @@ import org.apache.usergrid.persistence.graph.impl.stage.EdgeMetaRepair;
import org.apache.usergrid.persistence.graph.impl.stage.EdgeMetaRepairImpl;
import org.apache.usergrid.persistence.graph.impl.stage.NodeDeleteListener;
import org.apache.usergrid.persistence.graph.impl.stage.NodeDeleteListenerImpl;
+import org.apache.usergrid.persistence.graph.serialization.EdgeMetadataSerialization;
+import org.apache.usergrid.persistence.graph.serialization.EdgeSerialization;
+import org.apache.usergrid.persistence.graph.serialization.EdgesObservable;
+import org.apache.usergrid.persistence.graph.serialization.NodeSerialization;
+import org.apache.usergrid.persistence.graph.serialization.TargetIdObservable;
+import org.apache.usergrid.persistence.graph.serialization.impl.EdgeDataMigrationImpl;
+import org.apache.usergrid.persistence.graph.serialization.impl.EdgeMetadataSerializationProxyImpl;
+import org.apache.usergrid.persistence.graph.serialization.impl.EdgeMetadataSerializationV1Impl;
+import org.apache.usergrid.persistence.graph.serialization.impl.EdgeMetadataSerializationV2Impl;
+import org.apache.usergrid.persistence.graph.serialization.impl.EdgeSerializationImpl;
+import org.apache.usergrid.persistence.graph.serialization.impl.EdgesObservableImpl;
+import org.apache.usergrid.persistence.graph.serialization.impl.GraphManagerFactoryImpl;
+import org.apache.usergrid.persistence.graph.serialization.impl.NodeSerializationImpl;
+import org.apache.usergrid.persistence.graph.serialization.impl.TargetIdObservableImpl;
+import org.apache.usergrid.persistence.graph.serialization.impl.migration.GraphMigrationPlugin;
import org.apache.usergrid.persistence.graph.serialization.impl.shard.EdgeColumnFamilies;
import org.apache.usergrid.persistence.graph.serialization.impl.shard.EdgeShardSerialization;
import org.apache.usergrid.persistence.graph.serialization.impl.shard.EdgeShardStrategy;
@@ -67,6 +81,7 @@ import com.google.inject.Inject;
import com.google.inject.Key;
import com.google.inject.Provides;
import com.google.inject.Singleton;
+import com.google.inject.TypeLiteral;
import com.google.inject.multibindings.Multibinder;
@@ -85,8 +100,6 @@ public class GraphModule extends AbstractModule {
bind( GraphManagerFactory.class ).to(GraphManagerFactoryImpl.class);
- //bind(GraphManager.class).to(GraphManagerImpl.class );
-
bind(EdgesObservable.class).to(EdgesObservableImpl.class);
bind(TargetIdObservable.class).to(TargetIdObservableImpl.class);
@@ -95,8 +108,6 @@ public class GraphModule extends AbstractModule {
bind(EdgeMetadataSerialization.class).to(EdgeMetadataSerializationProxyImpl.class);
- bind(EdgeMigrationStrategy.class).to(EdgeMetadataSerializationProxyImpl.class);
-
/**
* bindings for shard allocations
*/
@@ -117,10 +128,19 @@ public class GraphModule extends AbstractModule {
bind( EdgeMetaRepair.class ).to( EdgeMetaRepairImpl.class );
bind( EdgeDeleteRepair.class ).to( EdgeDeleteRepairImpl.class );
- Multibinder<DataMigration> dataMigrationMultibinder =
- Multibinder.newSetBinder( binder(), DataMigration.class );
+
+ //wire up the edg migration
+ Multibinder<DataMigration2<ApplicationScope>> dataMigrationMultibinder =
+ Multibinder.newSetBinder( binder(), new TypeLiteral<DataMigration2<ApplicationScope>>() {} );
+
+
dataMigrationMultibinder.addBinding().to( EdgeDataMigrationImpl.class );
+
+ //wire up the collection migration plugin
+ Multibinder.newSetBinder( binder(), MigrationPlugin.class ).addBinding().to( GraphMigrationPlugin.class );
+
+
/**
* Add our listeners
*/
@@ -158,18 +178,17 @@ public class GraphModule extends AbstractModule {
migrationBinding.addBinding().to( Key.get( NodeShardCounterSerialization.class ) );
//Get the old version and the new one
- migrationBinding.addBinding().to( Key.get( EdgeMetadataSerialization.class, V1Impl.class) );
- migrationBinding.addBinding().to( Key.get( EdgeMetadataSerialization.class, V2Impl.class ) );
+ migrationBinding.addBinding().to( Key.get( EdgeMetadataSerializationV1Impl.class) );
+ migrationBinding.addBinding().to( Key.get( EdgeMetadataSerializationV2Impl.class ) );
/**
* Migrations of our edge meta serialization
*/
- bind(EdgeMetadataSerialization.class).annotatedWith( V1Impl.class ).to( EdgeMetadataSerializationV1Impl.class );
- bind(EdgeMetadataSerialization.class).annotatedWith( V2Impl.class ).to( EdgeMetadataSerializationV2Impl.class );
+ bind( EdgeMetadataSerializationV1Impl.class );
+ bind( EdgeMetadataSerializationV2Impl.class );
bind(EdgeMetadataSerialization.class).annotatedWith( ProxyImpl.class ).to( EdgeMetadataSerializationProxyImpl.class );
- bind(EdgeMigrationStrategy.class).annotatedWith(ProxyImpl.class).to( EdgeMetadataSerializationProxyImpl.class );
}
@@ -184,6 +203,35 @@ public class GraphModule extends AbstractModule {
}
+
+ /**
+ * Configure via explicit declaration the migration path we can follow
+ * @param v1
+ * @param v2
+ * @return
+ */
+ @Singleton
+ @Inject
+ @Provides
+ public VersionedMigrationSet<EdgeMetadataSerialization> getVersions(final EdgeMetadataSerializationV1Impl v1, final EdgeMetadataSerializationV2Impl v2){
+
+
+ //migrate from v1 to v2
+ MigrationRelationship<EdgeMetadataSerialization> v1Tov2 = new MigrationRelationship<>( v1, v2);
+
+
+ //keep our curent tuple, v2, v2
+ MigrationRelationship<EdgeMetadataSerialization> current = new MigrationRelationship<EdgeMetadataSerialization>( v2, v2 );
+
+
+ //now create our set of versions
+ VersionedMigrationSet<EdgeMetadataSerialization> set = new VersionedMigrationSet<>( v1Tov2, current );
+
+ return set;
+
+ }
+
+
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fa69be86/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/EdgeMetadataSerialization.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/EdgeMetadataSerialization.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/EdgeMetadataSerialization.java
index 38916ac..a843b3c 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/EdgeMetadataSerialization.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/EdgeMetadataSerialization.java
@@ -22,6 +22,7 @@ package org.apache.usergrid.persistence.graph.serialization;
import java.util.Iterator;
+import org.apache.usergrid.persistence.core.migration.data.newimpls.VersionedData;
import org.apache.usergrid.persistence.core.migration.schema.Migration;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.graph.Edge;
@@ -35,7 +36,7 @@ import com.netflix.astyanax.MutationBatch;
/**
* Simple interface for serializing an edge meta data
*/
-public interface EdgeMetadataSerialization extends Migration {
+public interface EdgeMetadataSerialization extends Migration, VersionedData {
/**
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fa69be86/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/EdgeMigrationStrategy.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/EdgeMigrationStrategy.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/EdgeMigrationStrategy.java
deleted file mode 100644
index 41fe7ad..0000000
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/EdgeMigrationStrategy.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- *
- * * Licensed to the Apache Software Foundation (ASF) under one or more
- * * contributor license agreements. The ASF licenses this file to You
- * * under the Apache License, Version 2.0 (the "License"); you may not
- * * use this file except in compliance with the License.
- * * You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing, software
- * * distributed under the License is distributed on an "AS IS" BASIS,
- * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * * See the License for the specific language governing permissions and
- * * limitations under the License. For additional information regarding
- * * copyright in this work, please see the NOTICE file in the top level
- * * directory of this distribution.
- *
- */
-package org.apache.usergrid.persistence.graph.serialization;
-
-import org.apache.usergrid.persistence.core.migration.schema.MigrationStrategy;
-import org.apache.usergrid.persistence.graph.Edge;
-
-import java.util.List;
-
-/**
- * Encapsulates version migration for graph
- */
-public interface EdgeMigrationStrategy extends MigrationStrategy<EdgeMetadataSerialization> {
- public static final int MIGRATION_VERSION = 2;
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fa69be86/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeDataMigrationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeDataMigrationImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeDataMigrationImpl.java
index 785e341..862c796 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeDataMigrationImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeDataMigrationImpl.java
@@ -25,11 +25,14 @@ import com.netflix.astyanax.MutationBatch;
import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
import org.apache.usergrid.persistence.core.migration.data.newimpls.DataMigration2;
import org.apache.usergrid.persistence.core.migration.data.newimpls.MigrationDataProvider;
+import org.apache.usergrid.persistence.core.migration.data.newimpls.MigrationRelationship;
+import org.apache.usergrid.persistence.core.migration.data.newimpls.ProgressObserver;
+import org.apache.usergrid.persistence.core.migration.data.newimpls.VersionedMigrationSet;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.graph.Edge;
import org.apache.usergrid.persistence.graph.GraphManager;
import org.apache.usergrid.persistence.graph.GraphManagerFactory;
-import org.apache.usergrid.persistence.graph.serialization.EdgeMigrationStrategy;
+import org.apache.usergrid.persistence.graph.serialization.EdgeMetadataSerialization;
import org.apache.usergrid.persistence.graph.serialization.EdgesObservable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -41,9 +44,8 @@ import java.util.List;
import java.util.concurrent.atomic.AtomicLong;
/**
- * Encapsulates data mi
+ * Encapsulates the migration of edge meta data
*/
-
public class EdgeDataMigrationImpl implements DataMigration2<ApplicationScope> {
private static final Logger logger = LoggerFactory.getLogger(EdgeDataMigrationImpl.class);
@@ -51,30 +53,35 @@ public class EdgeDataMigrationImpl implements DataMigration2<ApplicationScope> {
private final Keyspace keyspace;
private final GraphManagerFactory graphManagerFactory;
private final EdgesObservable edgesFromSourceObservable;
- private final EdgeMigrationStrategy edgeMigrationStrategy;
+ private final VersionedMigrationSet<EdgeMetadataSerialization> allVersions;
+ private final EdgeMetadataSerializationV2Impl edgeMetadataSerializationV2;
@Inject
- public EdgeDataMigrationImpl(final Keyspace keyspace,
- final GraphManagerFactory graphManagerFactory,
- final EdgesObservable edgesFromSourceObservable,
- final EdgeMigrationStrategy edgeMigrationStrategy
- ) {
+ public EdgeDataMigrationImpl( final Keyspace keyspace, final GraphManagerFactory graphManagerFactory,
+ final EdgesObservable edgesFromSourceObservable,
+
+ final VersionedMigrationSet<EdgeMetadataSerialization> allVersions,
+ final EdgeMetadataSerializationV2Impl edgeMetadataSerializationV2 ) {
this.keyspace = keyspace;
this.graphManagerFactory = graphManagerFactory;
this.edgesFromSourceObservable = edgesFromSourceObservable;
- this.edgeMigrationStrategy = edgeMigrationStrategy;
+ this.allVersions = allVersions;
+ this.edgeMetadataSerializationV2 = edgeMetadataSerializationV2;
}
-
@Override
- public void migrate( final MigrationDataProvider<ApplicationScope> migrationDataProvider,
- final ProgressObserver observer ) {
+ public int migrate( final int currentVersion, final MigrationDataProvider<ApplicationScope> migrationDataProvider,
+ final ProgressObserver observer ) {
+
final AtomicLong counter = new AtomicLong();
+ final MigrationRelationship<EdgeMetadataSerialization>
+ migration = allVersions.getMigrationRelationship( currentVersion );
+
migrationDataProvider.getData().flatMap(new Func1<ApplicationScope, Observable<?>>() {
@Override
public Observable call(final ApplicationScope applicationScope) {
@@ -99,8 +106,7 @@ public class EdgeDataMigrationImpl implements DataMigration2<ApplicationScope> {
for ( Edge edge : edges ) {
logger.info( "Migrating meta for edge {}", edge );
final MutationBatch edgeBatch =
- edgeMigrationStrategy.getMigration().to()
- .writeEdge( applicationScope, edge );
+ migration.to.writeEdge( applicationScope, edge );
batch.mergeShallow( edgeBatch );
}
@@ -114,7 +120,7 @@ public class EdgeDataMigrationImpl implements DataMigration2<ApplicationScope> {
//update the observer so the admin can see it
final long newCount = counter.addAndGet( edges.size() );
- observer.update( getVersion(),
+ observer.update( migration.to.getImplementationVersion(),
String.format( "Currently running. Rewritten %d edge types",
newCount ) );
}
@@ -124,12 +130,22 @@ public class EdgeDataMigrationImpl implements DataMigration2<ApplicationScope> {
}
});
+ return migration.to.getImplementationVersion();
+
}
+
+
@Override
- public int getVersion() {
- return edgeMigrationStrategy.getVersion();
+ public boolean supports( final int currentVersion ) {
+ return currentVersion <= edgeMetadataSerializationV2.getImplementationVersion();
}
+
+ @Override
+ public int getMaxVersion() {
+ //we only support up to v2 ATM
+ return edgeMetadataSerializationV2.getImplementationVersion();
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fa69be86/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeMetadataSerializationProxyImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeMetadataSerializationProxyImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeMetadataSerializationProxyImpl.java
index acba53f..a5bde21 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeMetadataSerializationProxyImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeMetadataSerializationProxyImpl.java
@@ -27,13 +27,15 @@ import java.util.Collections;
import java.util.Iterator;
import org.apache.usergrid.persistence.core.astyanax.MultiTennantColumnFamilyDefinition;
-import org.apache.usergrid.persistence.core.guice.V1Impl;
-import org.apache.usergrid.persistence.core.guice.V2Impl;
import org.apache.usergrid.persistence.core.migration.data.DataMigrationManager;
+import org.apache.usergrid.persistence.core.migration.data.MigrationInfoCache;
+import org.apache.usergrid.persistence.core.migration.data.newimpls.MigrationRelationship;
+import org.apache.usergrid.persistence.core.migration.data.newimpls.VersionedMigrationSet;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.graph.*;
import org.apache.usergrid.persistence.graph.serialization.EdgeMetadataSerialization;
import org.apache.usergrid.persistence.graph.serialization.EdgeMigrationStrategy;
+import org.apache.usergrid.persistence.graph.serialization.impl.migration.GraphMigrationPlugin;
import org.apache.usergrid.persistence.model.entity.Id;
import com.google.inject.Inject;
@@ -49,90 +51,93 @@ public class EdgeMetadataSerializationProxyImpl implements EdgeMetadataSerializa
private static final Logger logger = LoggerFactory.getLogger(EdgeMetadataSerializationProxyImpl.class);
- private final DataMigrationManager dataMigrationManager;
private final Keyspace keyspace;
- private final EdgeMetadataSerialization previous;
- private final EdgeMetadataSerialization current;
+ private final VersionedMigrationSet<EdgeMetadataSerialization> versions;
+ private final MigrationInfoCache migrationInfoCache;
/**
* Handles routing data to the right implementation, based on the current system migration version
*/
@Inject
- public EdgeMetadataSerializationProxyImpl(final DataMigrationManager dataMigrationManager, final Keyspace keyspace,
- @V1Impl final EdgeMetadataSerialization previous,
- @V2Impl final EdgeMetadataSerialization current) {
- this.dataMigrationManager = dataMigrationManager;
+ public EdgeMetadataSerializationProxyImpl( final Keyspace keyspace,
+ final VersionedMigrationSet<EdgeMetadataSerialization> versions,
+ final MigrationInfoCache migrationInfoCache ) {
this.keyspace = keyspace;
- this.previous = previous;
- this.current = current;
+ this.versions = versions;
+ this.migrationInfoCache = migrationInfoCache;
}
@Override
public MutationBatch writeEdge( final ApplicationScope scope, final Edge edge ) {
+ final MigrationRelationship<EdgeMetadataSerialization> migration = getMigrationRelationShip();
- if ( isOldVersion() ) {
+
+ if ( migration.needsMigration() ) {
final MutationBatch aggregateBatch = keyspace.prepareMutationBatch();
- aggregateBatch.mergeShallow( previous.writeEdge( scope, edge ) );
- aggregateBatch.mergeShallow( current.writeEdge( scope, edge ) );
+ aggregateBatch.mergeShallow( migration.from.writeEdge( scope, edge ) );
+ aggregateBatch.mergeShallow( migration.to.writeEdge( scope, edge ) );
return aggregateBatch;
}
- return current.writeEdge( scope, edge );
+ return migration.to.writeEdge( scope, edge );
}
@Override
public MutationBatch removeEdgeTypeFromSource( final ApplicationScope scope, final Edge edge ) {
+ final MigrationRelationship<EdgeMetadataSerialization> migration = getMigrationRelationShip();
- if ( isOldVersion() ) {
+ if ( migration.needsMigration() ) {
final MutationBatch aggregateBatch = keyspace.prepareMutationBatch();
- aggregateBatch.mergeShallow( previous.removeEdgeTypeFromSource( scope, edge ) );
- aggregateBatch.mergeShallow( current.removeEdgeTypeFromSource( scope, edge ) );
+ aggregateBatch.mergeShallow( migration.from.removeEdgeTypeFromSource( scope, edge ) );
+ aggregateBatch.mergeShallow( migration.to.removeEdgeTypeFromSource( scope, edge ) );
return aggregateBatch;
}
- return current.removeEdgeTypeFromSource( scope, edge );
+ return migration.to.removeEdgeTypeFromSource( scope, edge );
}
@Override
public MutationBatch removeEdgeTypeFromSource( final ApplicationScope scope, final Id sourceNode, final String type,
final long timestamp ) {
+ final MigrationRelationship<EdgeMetadataSerialization> migration = getMigrationRelationShip();
-
- if ( isOldVersion() ) {
+ if ( migration.needsMigration() ) {
final MutationBatch aggregateBatch = keyspace.prepareMutationBatch();
- aggregateBatch.mergeShallow( previous.removeEdgeTypeFromSource( scope, sourceNode, type, timestamp ) );
- aggregateBatch.mergeShallow( current.removeEdgeTypeFromSource( scope, sourceNode, type, timestamp ) );
+ aggregateBatch.mergeShallow( migration.from.removeEdgeTypeFromSource( scope, sourceNode, type, timestamp ) );
+ aggregateBatch.mergeShallow( migration.to.removeEdgeTypeFromSource( scope, sourceNode, type, timestamp ) );
return aggregateBatch;
}
- return current.removeEdgeTypeFromSource( scope, sourceNode, type, timestamp );
+ return migration.to.removeEdgeTypeFromSource( scope, sourceNode, type, timestamp );
}
@Override
public MutationBatch removeIdTypeFromSource( final ApplicationScope scope, final Edge edge ) {
- if ( isOldVersion() ) {
+ final MigrationRelationship<EdgeMetadataSerialization> migration = getMigrationRelationShip();
+
+ if ( migration.needsMigration() ) {
final MutationBatch aggregateBatch = keyspace.prepareMutationBatch();
- aggregateBatch.mergeShallow( previous.removeIdTypeFromSource( scope, edge ) );
- aggregateBatch.mergeShallow( current.removeIdTypeFromSource( scope, edge ) );
+ aggregateBatch.mergeShallow( migration.from.removeIdTypeFromSource( scope, edge ) );
+ aggregateBatch.mergeShallow( migration.to.removeIdTypeFromSource( scope, edge ) );
return aggregateBatch;
}
- return current.removeIdTypeFromSource( scope, edge );
+ return migration.to.removeIdTypeFromSource( scope, edge );
}
@@ -140,34 +145,37 @@ public class EdgeMetadataSerializationProxyImpl implements EdgeMetadataSerializa
public MutationBatch removeIdTypeFromSource( final ApplicationScope scope, final Id sourceNode, final String type,
final String idType, final long timestamp ) {
- if ( isOldVersion() ) {
+ final MigrationRelationship<EdgeMetadataSerialization> migration = getMigrationRelationShip();
+
+ if ( migration.needsMigration() ) {
final MutationBatch aggregateBatch = keyspace.prepareMutationBatch();
aggregateBatch
- .mergeShallow( previous.removeIdTypeFromSource( scope, sourceNode, type, idType, timestamp ) );
- aggregateBatch.mergeShallow( current.removeIdTypeFromSource( scope, sourceNode, type, idType, timestamp ) );
+ .mergeShallow( migration.from.removeIdTypeFromSource( scope, sourceNode, type, idType, timestamp ) );
+ aggregateBatch.mergeShallow( migration.to.removeIdTypeFromSource( scope, sourceNode, type, idType, timestamp ) );
return aggregateBatch;
}
- return current.removeIdTypeFromSource( scope, sourceNode, type, idType, timestamp );
+ return migration.to.removeIdTypeFromSource( scope, sourceNode, type, idType, timestamp );
}
@Override
public MutationBatch removeEdgeTypeToTarget( final ApplicationScope scope, final Edge edge ) {
+ final MigrationRelationship<EdgeMetadataSerialization> migration = getMigrationRelationShip();
- if ( isOldVersion() ) {
+ if ( migration.needsMigration() ) {
final MutationBatch aggregateBatch = keyspace.prepareMutationBatch();
- aggregateBatch.mergeShallow( previous.removeEdgeTypeToTarget( scope, edge ) );
- aggregateBatch.mergeShallow( current.removeEdgeTypeToTarget( scope, edge ) );
+ aggregateBatch.mergeShallow( migration.from.removeEdgeTypeToTarget( scope, edge ) );
+ aggregateBatch.mergeShallow( migration.to.removeEdgeTypeToTarget( scope, edge ) );
return aggregateBatch;
}
- return current.removeEdgeTypeToTarget( scope, edge );
+ return migration.to.removeEdgeTypeToTarget( scope, edge );
}
@@ -175,90 +183,102 @@ public class EdgeMetadataSerializationProxyImpl implements EdgeMetadataSerializa
public MutationBatch removeEdgeTypeToTarget( final ApplicationScope scope, final Id targetNode, final String type,
final long timestamp ) {
- if ( isOldVersion() ) {
+ final MigrationRelationship<EdgeMetadataSerialization> migration = getMigrationRelationShip();
+
+ if ( migration.needsMigration() ) {
final MutationBatch aggregateBatch = keyspace.prepareMutationBatch();
- aggregateBatch.mergeShallow( previous.removeEdgeTypeToTarget( scope, targetNode, type, timestamp ) );
- aggregateBatch.mergeShallow( current.removeEdgeTypeToTarget( scope, targetNode, type, timestamp ) );
+ aggregateBatch.mergeShallow( migration.from.removeEdgeTypeToTarget( scope, targetNode, type, timestamp ) );
+ aggregateBatch.mergeShallow( migration.to.removeEdgeTypeToTarget( scope, targetNode, type, timestamp ) );
return aggregateBatch;
}
- return current.removeEdgeTypeToTarget( scope, targetNode, type, timestamp );
+ return migration.to.removeEdgeTypeToTarget( scope, targetNode, type, timestamp );
}
@Override
public MutationBatch removeIdTypeToTarget( final ApplicationScope scope, final Edge edge ) {
- if ( isOldVersion() ) {
+ final MigrationRelationship<EdgeMetadataSerialization> migration = getMigrationRelationShip();
+
+ if ( migration.needsMigration() ) {
final MutationBatch aggregateBatch = keyspace.prepareMutationBatch();
- aggregateBatch.mergeShallow( previous.removeIdTypeToTarget( scope, edge ) );
- aggregateBatch.mergeShallow( current.removeIdTypeToTarget( scope, edge ) );
+ aggregateBatch.mergeShallow( migration.from.removeIdTypeToTarget( scope, edge ) );
+ aggregateBatch.mergeShallow( migration.to.removeIdTypeToTarget( scope, edge ) );
return aggregateBatch;
}
- return current.removeIdTypeToTarget( scope, edge );
+ return migration.to.removeIdTypeToTarget( scope, edge );
}
@Override
public MutationBatch removeIdTypeToTarget( final ApplicationScope scope, final Id targetNode, final String type,
final String idType, final long timestamp ) {
+ final MigrationRelationship<EdgeMetadataSerialization> migration = getMigrationRelationShip();
-
- if ( isOldVersion() ) {
+ if ( migration.needsMigration() ) {
final MutationBatch aggregateBatch = keyspace.prepareMutationBatch();
- aggregateBatch.mergeShallow( previous.removeIdTypeToTarget( scope, targetNode, type, idType, timestamp ) );
- aggregateBatch.mergeShallow( current.removeIdTypeToTarget( scope, targetNode, type, idType, timestamp ) );
+ aggregateBatch.mergeShallow( migration.from.removeIdTypeToTarget( scope, targetNode, type, idType, timestamp ) );
+ aggregateBatch.mergeShallow( migration.to.removeIdTypeToTarget( scope, targetNode, type, idType, timestamp ) );
return aggregateBatch;
}
- return current.removeIdTypeToTarget( scope, targetNode, type, idType, timestamp );
+ return migration.to.removeIdTypeToTarget( scope, targetNode, type, idType, timestamp );
}
@Override
public Iterator<String> getEdgeTypesFromSource( final ApplicationScope scope, final SearchEdgeType search ) {
- if ( isOldVersion() ) {
- return previous.getEdgeTypesFromSource( scope, search );
+ final MigrationRelationship<EdgeMetadataSerialization> migration = getMigrationRelationShip();
+
+ if ( migration.needsMigration() ) {
+ return migration.from.getEdgeTypesFromSource( scope, search );
}
- return current.getEdgeTypesFromSource( scope, search );
+ return migration.to.getEdgeTypesFromSource( scope, search );
}
@Override
public Iterator<String> getIdTypesFromSource( final ApplicationScope scope, final SearchIdType search ) {
- if ( isOldVersion() ) {
- return previous.getIdTypesFromSource( scope, search );
+ final MigrationRelationship<EdgeMetadataSerialization> migration = getMigrationRelationShip();
+
+ if ( migration.needsMigration() ) {
+ return migration.from.getIdTypesFromSource( scope, search );
}
- return current.getIdTypesFromSource( scope, search );
+ return migration.to.getIdTypesFromSource( scope, search );
}
@Override
public Iterator<String> getEdgeTypesToTarget( final ApplicationScope scope, final SearchEdgeType search ) {
- if ( isOldVersion() ) {
- return previous.getEdgeTypesToTarget( scope, search );
+ final MigrationRelationship<EdgeMetadataSerialization> migration = getMigrationRelationShip();
+
+ if ( migration.needsMigration() ) {
+ return migration.from.getEdgeTypesToTarget( scope, search );
}
- return current.getEdgeTypesToTarget( scope, search );
+ return migration.to.getEdgeTypesToTarget( scope, search );
}
@Override
public Iterator<String> getIdTypesToTarget( final ApplicationScope scope, final SearchIdType search ) {
- if ( isOldVersion() ) {
- return previous.getIdTypesToTarget( scope, search );
+ final MigrationRelationship<EdgeMetadataSerialization> migration = getMigrationRelationShip();
+
+ if ( migration.needsMigration() ) {
+ return migration.from.getIdTypesToTarget( scope, search );
}
- return current.getIdTypesToTarget( scope, search );
+ return migration.to.getIdTypesToTarget( scope, search );
}
@@ -268,21 +288,19 @@ public class EdgeMetadataSerializationProxyImpl implements EdgeMetadataSerializa
}
+
+
/**
* Return true if we're on an old version
*/
- private boolean isOldVersion() {
- return dataMigrationManager.getCurrentVersion() < getVersion();
+ private MigrationRelationship<EdgeMetadataSerialization> getMigrationRelationShip() {
+ return this.versions.getMigrationRelationship(
+ migrationInfoCache.getVersion( GraphMigrationPlugin.PLUGIN_NAME ) );
}
- @Override
- public MigrationRelationship<EdgeMetadataSerialization> getMigration() {
- return new MigrationRelationship<>(previous,current);
- }
@Override
- public int getVersion() {
- return MIGRATION_VERSION;
+ public int getImplementationVersion() {
+ throw new UnsupportedOperationException( "Proxies do not have an implementation version" );
}
-
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fa69be86/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeMetadataSerializationV1Impl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeMetadataSerializationV1Impl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeMetadataSerializationV1Impl.java
index c634684..5fd270f 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeMetadataSerializationV1Impl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeMetadataSerializationV1Impl.java
@@ -387,6 +387,12 @@ public class EdgeMetadataSerializationV1Impl implements EdgeMetadataSerializatio
}
+ @Override
+ public int getImplementationVersion() {
+ return GraphDataVersions.INITIAL.getVersion();
+ }
+
+
/**
* Inner class to serialize and edgeIdTypeKey
*/
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fa69be86/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeMetadataSerializationV2Impl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeMetadataSerializationV2Impl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeMetadataSerializationV2Impl.java
index c9a279a..9e59dbf 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeMetadataSerializationV2Impl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeMetadataSerializationV2Impl.java
@@ -536,6 +536,12 @@ public class EdgeMetadataSerializationV2Impl implements EdgeMetadataSerializatio
}
+ @Override
+ public int getImplementationVersion() {
+ return GraphDataVersions.META_SHARDING.getVersion();
+ }
+
+
/**
* Inner class to serialize and edgeIdTypeKey
*/
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fa69be86/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/GraphDataVersions.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/GraphDataVersions.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/GraphDataVersions.java
new file mode 100644
index 0000000..60d36bd
--- /dev/null
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/GraphDataVersions.java
@@ -0,0 +1,43 @@
+/*
+ *
+ * *
+ * * Licensed to the Apache Software Foundation (ASF) under one
+ * * or more contributor license agreements. See the NOTICE file
+ * * distributed with this work for additional information
+ * * regarding copyright ownership. The ASF licenses this file
+ * * to you under the Apache License, Version 2.0 (the
+ * * "License"); you may not use this file except in compliance
+ * * with the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing,
+ * * software distributed under the License is distributed on an
+ * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * * KIND, either express or implied. See the License for the
+ * * specific language governing permissions and limitations
+ * * under the License.
+ * *
+ *
+ */
+
+package org.apache.usergrid.persistence.graph.serialization.impl;
+
+
+/**
+ * Versions of data as they exist across our system
+ */
+public enum GraphDataVersions {
+ INITIAL(0),
+ META_SHARDING(1);
+
+ private final int version;
+
+
+ private GraphDataVersions( final int version ) {this.version = version;}
+
+
+ public int getVersion() {
+ return version;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fa69be86/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/GraphMigrationPlugin.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/GraphMigrationPlugin.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/GraphMigrationPlugin.java
new file mode 100644
index 0000000..0d3405b
--- /dev/null
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/GraphMigrationPlugin.java
@@ -0,0 +1,63 @@
+/*
+ *
+ * *
+ * * Licensed to the Apache Software Foundation (ASF) under one
+ * * or more contributor license agreements. See the NOTICE file
+ * * distributed with this work for additional information
+ * * regarding copyright ownership. The ASF licenses this file
+ * * to you under the Apache License, Version 2.0 (the
+ * * "License"); you may not use this file except in compliance
+ * * with the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing,
+ * * software distributed under the License is distributed on an
+ * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * * KIND, either express or implied. See the License for the
+ * * specific language governing permissions and limitations
+ * * under the License.
+ * *
+ *
+ */
+
+package org.apache.usergrid.persistence.graph.serialization.impl.migration;
+
+
+import java.util.Set;
+
+import org.apache.usergrid.persistence.core.migration.data.MigrationInfoSerialization;
+import org.apache.usergrid.persistence.core.migration.data.newimpls.AbstractMigrationPlugin;
+import org.apache.usergrid.persistence.core.migration.data.newimpls.DataMigration2;
+import org.apache.usergrid.persistence.core.migration.data.newimpls.MigrationDataProvider;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+
+
+/**
+ * Migration plugin for the collection module
+ */
+@Singleton
+public class GraphMigrationPlugin extends AbstractMigrationPlugin<ApplicationScope> {
+
+ public static final String PLUGIN_NAME = "graph-data";
+
+
+
+ @Inject
+ public GraphMigrationPlugin( final Set<DataMigration2<ApplicationScope>> entityDataMigrations,
+ final MigrationDataProvider<ApplicationScope> entityIdScopeDataMigrationProvider,
+ final MigrationInfoSerialization migrationInfoSerialization ) {
+ super( entityDataMigrations, entityIdScopeDataMigrationProvider, migrationInfoSerialization );
+ }
+
+
+ @Override
+ public String getName() {
+ return PLUGIN_NAME;
+ }
+
+
+}
[16/50] [abbrv] incubator-usergrid git commit: Temporarily removed
test. Need to rebuild with plugin structure.
Posted by sf...@apache.org.
Temporarily removed test. Need to rebuild with plugin structure.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/12c50359
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/12c50359
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/12c50359
Branch: refs/heads/USERGRID-480
Commit: 12c503591cb1d37301533d95075fe1d27d5bbf39
Parents: d3ce4e7
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Mar 4 11:53:24 2015 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Mar 4 11:53:24 2015 -0700
----------------------------------------------------------------------
.../migration/EntityTypeMappingMigrationIT.java | 211 +++++++++----------
.../migration/MigrationTestRule.java | 98 ---------
2 files changed, 95 insertions(+), 214 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/12c50359/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/EntityTypeMappingMigrationIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/EntityTypeMappingMigrationIT.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/EntityTypeMappingMigrationIT.java
index 99fb139..35608d5 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/EntityTypeMappingMigrationIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/EntityTypeMappingMigrationIT.java
@@ -20,42 +20,20 @@
package org.apache.usergrid.corepersistence.migration;
-import java.util.HashSet;
-import java.util.Set;
-
-import org.apache.usergrid.persistence.collection.CollectionScope;
-import org.apache.usergrid.persistence.core.rx.AllEntitiesInSystemObservable;
-import org.apache.usergrid.persistence.core.scope.ApplicationEntityGroup;
-import org.apache.usergrid.persistence.core.scope.EntityIdScope;
import org.junit.Before;
import org.junit.Ignore;
-import org.junit.Rule;
import org.junit.Test;
import org.apache.usergrid.AbstractCoreIT;
-import org.apache.usergrid.cassandra.SpringResource;
-import org.apache.usergrid.corepersistence.EntityWriteHelper;
import org.apache.usergrid.corepersistence.ManagerCache;
-import org.apache.usergrid.corepersistence.util.CpNamingUtils;
-import org.apache.usergrid.persistence.Entity;
-import org.apache.usergrid.persistence.EntityManager;
import org.apache.usergrid.persistence.EntityManagerFactory;
import org.apache.usergrid.persistence.core.migration.data.DataMigrationManager;
-import org.apache.usergrid.persistence.map.impl.MapSerializationImpl;
-import org.apache.usergrid.persistence.model.entity.Id;
-import org.apache.usergrid.persistence.model.util.UUIDGenerator;
import com.google.inject.Injector;
import com.netflix.astyanax.Keyspace;
import net.jcip.annotations.NotThreadSafe;
-import rx.functions.Action1;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
@NotThreadSafe
public class EntityTypeMappingMigrationIT extends AbstractCoreIT {
@@ -71,24 +49,25 @@ public class EntityTypeMappingMigrationIT extends AbstractCoreIT {
private DataMigrationManager dataMigrationManager;
- /**
- * Rule to do the resets we need
- */
- @Rule
- public MigrationTestRule migrationTestRule = new MigrationTestRule(
- app, SpringResource.getInstance().getBean( Injector.class ) ,EntityTypeMappingMigration.class );
- private AllEntitiesInSystemObservable allEntitiesInSystemObservable;
+// /**
+// * Rule to do the resets we need
+// */
+// @Rule
+// public MigrationTestRule migrationTestRule = new MigrationTestRule(
+// app, SpringResource.getInstance().getBean( Injector.class ) ,EntityTypeMappingMigration.class );
+// private AllEntitiesInSystemObservable allEntitiesInSystemObservable;
+ //TODO USERGRID-405
@Before
public void setup() {
- injector = SpringResource.getInstance().getBean( Injector.class );
- emf = setup.getEmf();
- entityTypeMappingMigration = injector.getInstance( EntityTypeMappingMigration.class );
- keyspace = injector.getInstance( Keyspace.class );
- managerCache = injector.getInstance( ManagerCache.class );
- dataMigrationManager = injector.getInstance( DataMigrationManager.class );
- allEntitiesInSystemObservable = injector.getInstance(AllEntitiesInSystemObservable.class);
+// injector = SpringResource.getInstance().getBean( Injector.class );
+// emf = setup.getEmf();
+// entityTypeMappingMigration = injector.getInstance( EntityTypeMappingMigration.class );
+// keyspace = injector.getInstance( Keyspace.class );
+// managerCache = injector.getInstance( ManagerCache.class );
+// dataMigrationManager = injector.getInstance( DataMigrationManager.class );
+// allEntitiesInSystemObservable = injector.getInstance(AllEntitiesInSystemObservable.class);
}
@@ -96,85 +75,85 @@ public class EntityTypeMappingMigrationIT extends AbstractCoreIT {
@Ignore("Ignored awaiting fix for USERGRID-268")
public void testIdMapping() throws Throwable {
- assertEquals( "version 1 expected", 1, entityTypeMappingMigration.getVersion() );
- assertEquals( "Previous version expected", 0, dataMigrationManager.getCurrentVersion());
-
- final EntityManager newAppEm = app.getEntityManager();
-
- final String type1 = "type1thing";
- final String type2 = "type2thing";
- final int size = 10;
-
- final Set<Id> type1Identities = EntityWriteHelper.createTypes( newAppEm, type1, size );
- final Set<Id> type2Identities = EntityWriteHelper.createTypes( newAppEm, type2, size );
-
-
- final Set<Id> allEntities = new HashSet<>();
- allEntities.addAll( type1Identities );
- allEntities.addAll( type2Identities );
-
-
- /**
- * Drop our map keyspace to ensure we have no entries before migrating after doing our writes.
- * This will ensure we have the data
- */
- keyspace.truncateColumnFamily( MapSerializationImpl.MAP_ENTRIES );
- keyspace.truncateColumnFamily( MapSerializationImpl.MAP_KEYS );
-
- app.createApplication(
- GraphShardVersionMigrationIT.class.getSimpleName()+ UUIDGenerator.newTimeUUID(),
- "migrationTest" );
-
-
-
- final TestProgressObserver progressObserver = new TestProgressObserver();
-
- entityTypeMappingMigration.migrate(allEntitiesInSystemObservable.getAllEntitiesInSystem(1000), progressObserver).toBlocking().last();
-
- allEntitiesInSystemObservable.getAllEntitiesInSystem(1000)
- .doOnNext(new Action1<ApplicationEntityGroup<CollectionScope>>() {
- @Override
- public void call(
- final ApplicationEntityGroup<CollectionScope> entity) {
- //ensure that each one has a type
-
- final EntityManager em = emf.getEntityManager(
- entity.applicationScope.getApplication().getUuid());
-
- for (final EntityIdScope<CollectionScope> idScope : entity.entityIds) {
- final Id id = idScope.getId();
- try {
- final Entity returned = em.get(id.getUuid());
-
- //we seem to occasionally get phantom edges. If this is the
- // case we'll store the type _> uuid mapping, but we won't have
- // anything to load
-
- if (returned != null) {
- assertEquals(id.getUuid(), returned.getUuid());
- assertEquals(id.getType(), returned.getType());
- } else {
- final String type = managerCache.getMapManager(CpNamingUtils
- .getEntityTypeMapScope(
- entity.applicationScope.getApplication()))
- .getString(id.getUuid()
- .toString());
-
- assertEquals(id.getType(), type);
- }
- } catch (Exception e) {
- throw new RuntimeException("Unable to get entity " + id
- + " by UUID, migration failed", e);
- }
-
- allEntities.remove(id);
- }
- }
- }).toBlocking().lastOrDefault(null);
-
-
- assertEquals("Every element should have been encountered", 0, allEntities.size());
- assertFalse("Progress observer should not have failed", progressObserver.getFailed());
- assertTrue("Progress observer should have update messages", progressObserver.getUpdates().size() > 0);
+// assertEquals( "version 1 expected", 1, entityTypeMappingMigration.getVersion() );
+// assertEquals( "Previous version expected", 0, dataMigrationManager.getCurrentVersion());
+//
+// final EntityManager newAppEm = app.getEntityManager();
+//
+// final String type1 = "type1thing";
+// final String type2 = "type2thing";
+// final int size = 10;
+//
+// final Set<Id> type1Identities = EntityWriteHelper.createTypes( newAppEm, type1, size );
+// final Set<Id> type2Identities = EntityWriteHelper.createTypes( newAppEm, type2, size );
+//
+//
+// final Set<Id> allEntities = new HashSet<>();
+// allEntities.addAll( type1Identities );
+// allEntities.addAll( type2Identities );
+//
+//
+// /**
+// * Drop our map keyspace to ensure we have no entries before migrating after doing our writes.
+// * This will ensure we have the data
+// */
+// keyspace.truncateColumnFamily( MapSerializationImpl.MAP_ENTRIES );
+// keyspace.truncateColumnFamily( MapSerializationImpl.MAP_KEYS );
+//
+// app.createApplication(
+// GraphShardVersionMigrationIT.class.getSimpleName()+ UUIDGenerator.newTimeUUID(),
+// "migrationTest" );
+//
+//
+//
+// final TestProgressObserver progressObserver = new TestProgressObserver();
+//
+// entityTypeMappingMigration.migrate(allEntitiesInSystemObservable.getAllEntitiesInSystem(1000), progressObserver).toBlocking().last();
+//
+// allEntitiesInSystemObservable.getAllEntitiesInSystem(1000)
+// .doOnNext(new Action1<ApplicationEntityGroup<CollectionScope>>() {
+// @Override
+// public void call(
+// final ApplicationEntityGroup<CollectionScope> entity) {
+// //ensure that each one has a type
+//
+// final EntityManager em = emf.getEntityManager(
+// entity.applicationScope.getApplication().getUuid());
+//
+// for (final EntityIdScope<CollectionScope> idScope : entity.entityIds) {
+// final Id id = idScope.getId();
+// try {
+// final Entity returned = em.get(id.getUuid());
+//
+// //we seem to occasionally get phantom edges. If this is the
+// // case we'll store the type _> uuid mapping, but we won't have
+// // anything to load
+//
+// if (returned != null) {
+// assertEquals(id.getUuid(), returned.getUuid());
+// assertEquals(id.getType(), returned.getType());
+// } else {
+// final String type = managerCache.getMapManager(CpNamingUtils
+// .getEntityTypeMapScope(
+// entity.applicationScope.getApplication()))
+// .getString(id.getUuid()
+// .toString());
+//
+// assertEquals(id.getType(), type);
+// }
+// } catch (Exception e) {
+// throw new RuntimeException("Unable to get entity " + id
+// + " by UUID, migration failed", e);
+// }
+//
+// allEntities.remove(id);
+// }
+// }
+// }).toBlocking().lastOrDefault(null);
+//
+//
+// assertEquals("Every element should have been encountered", 0, allEntities.size());
+// assertFalse("Progress observer should not have failed", progressObserver.getFailed());
+// assertTrue("Progress observer should have update messages", progressObserver.getUpdates().size() > 0);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/12c50359/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/MigrationTestRule.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/MigrationTestRule.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/MigrationTestRule.java
deleted file mode 100644
index 67f89ab..0000000
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/MigrationTestRule.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.migration;
-
-
-import org.junit.rules.ExternalResource;
-import org.junit.runner.Description;
-import org.junit.runners.model.Statement;
-
-import org.apache.usergrid.CoreApplication;
-import org.apache.usergrid.persistence.core.migration.data.DataMigrationManager;
-import org.apache.usergrid.persistence.core.migration.data.newimpls.DataMigration2;
-import org.apache.usergrid.persistence.core.migration.data.newimpls.MigrationPlugin;
-import org.apache.usergrid.persistence.model.util.UUIDGenerator;
-
-import com.google.inject.Injector;
-
-
-/**
- * This class is required because we cannot create the system's framework on a higher version (the default behavior is
- * the latest) then roll back to a previous version
- *
- * This rule performs the following operations.
- *
- * <ol> <li>Sets up the system's version to be the DataMigration impl's version -1</li> <li>Creates an application using
- * that version of the schema.</li> <li>Sets the new org and app in the CoreApplication to the app created</li> </ol>
- */
-public class MigrationTestRule extends ExternalResource {
-
- protected final CoreApplication core;
- protected final DataMigrationManager dataMigrationManager;
- protected final DataMigration2 dataMigration;
-
- protected int currentVersion;
-
-
- /**
- * Create a new migration test rule.
- *
- * @param core the CoreApplication rule used in this test
- * @param injector The injector used in this test
- */
- public MigrationTestRule( final CoreApplication core, final Injector injector ) {
- this.core = core;
- this.dataMigrationManager = injector.getInstance( DataMigrationManager.class );
- this.dataMigration = injector.getInstance( dataMigrationClass );
- }
-
-
- public void resetAndCreateApp( final String className, final String methodName ) throws Exception {
- dataMigrationManager.invalidate();
- currentVersion = dataMigrationManager.getCurrentVersion();
-
- dataMigrationManager.resetToVersion( dataMigration.getVersion() - 1 );
- dataMigrationManager.invalidate();
-
- core.createApplication( className + UUIDGenerator.newTimeUUID(), methodName );
- }
-
-
- public void resetVersion(){
- dataMigrationManager.resetToVersion( currentVersion );
- dataMigrationManager.invalidate();
- }
-
- @Override
- public Statement apply( final Statement base, final Description description ) {
-
- return new Statement() {
- @Override
- public void evaluate() throws Throwable {
- try {
- resetAndCreateApp( description.getClassName(), description.getMethodName() );
- base.evaluate();
- }finally {
- resetVersion();
- }
- }
- };
- }
-}
[17/50] [abbrv] incubator-usergrid git commit: Merge remote-tracking
branch 'origin/two-dot-o' into USERGRID-405
Posted by sf...@apache.org.
Merge remote-tracking branch 'origin/two-dot-o' into USERGRID-405
Conflicts:
stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
stack/core/src/test/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImplIT.java
stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/guice/CommonModule.java
stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
stack/rest/src/main/java/org/apache/usergrid/rest/IndexResource.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/08719b72
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/08719b72
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/08719b72
Branch: refs/heads/USERGRID-480
Commit: 08719b72727d2557e089284dc7bf28b2a8cf1afb
Parents: 12c5035 c1f5b78
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Mar 4 14:24:45 2015 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Mar 4 14:24:45 2015 -0700
----------------------------------------------------------------------
.../src/main/groovy/configure_usergrid.groovy | 2 +-
.../main/resources/usergrid-default.properties | 15 +-
.../batch/service/JobSchedulerService.java | 2 +-
.../corepersistence/CpEntityManager.java | 80 +-
.../corepersistence/CpEntityManagerFactory.java | 148 ++-
.../corepersistence/CpRelationManager.java | 6 +-
.../usergrid/corepersistence/CpSetup.java | 57 +-
.../usergrid/corepersistence/CpWalker.java | 119 ++-
.../usergrid/corepersistence/GuiceFactory.java | 25 +-
.../results/FilteringLoader.java | 42 +-
.../corepersistence/util/CpNamingUtils.java | 3 +
.../apache/usergrid/metrics/MetricsFactory.java | 112 ---
.../usergrid/persistence/EntityManager.java | 3 +
.../persistence/EntityManagerFactory.java | 22 +-
.../persistence/cassandra/CassandraService.java | 71 +-
.../cassandra/EntityManagerFactoryImpl.java | 43 +-
.../cassandra/EntityManagerImpl.java | 7 +
.../persistence/cassandra/SetupImpl.java | 60 +-
.../entities/FailedImportEntity.java | 11 +-
.../main/resources/usergrid-core-context.xml | 9 +-
.../corepersistence/StaleIndexCleanupTest.java | 34 +-
.../count/BatchCountParallelismTest.java | 2 +
.../usergrid/persistence/CollectionIT.java | 2 +-
.../usergrid/persistence/CoreSchemaManager.java | 4 +-
.../PerformanceEntityRebuildIndexTest.java | 192 +++-
.../cassandra/EntityManagerFactoryImplIT.java | 90 +-
.../apache/usergrid/utils/UUIDUtilsTest.java | 76 +-
.../impl/EntityCollectionManagerImpl.java | 7 +-
.../src/test/resources/log4j.properties | 2 +-
stack/corepersistence/common/pom.xml | 12 +
.../core/astyanax/AstyanaxKeyspaceProvider.java | 2 +-
.../persistence/core/astyanax/CassandraFig.java | 10 +-
.../persistence/core/future/BetterFuture.java | 43 +
.../persistence/core/guice/CommonModule.java | 6 +-
.../core/metrics/MetricsFactory.java | 34 +
.../core/metrics/MetricsFactoryImpl.java | 113 +++
.../persistence/core/metrics/MetricsFig.java | 33 +
.../common/src/test/resources/log4j.properties | 6 +-
.../graph/src/test/resources/log4j.properties | 8 +-
stack/corepersistence/pom.xml | 1 +
.../usergrid/persistence/index/EntityIndex.java | 9 +-
.../persistence/index/EntityIndexBatch.java | 32 +-
.../persistence/index/IndexBatchBuffer.java | 36 +
.../persistence/index/IndexBufferConsumer.java | 26 +
.../persistence/index/IndexBufferProducer.java | 36 +
.../usergrid/persistence/index/IndexFig.java | 45 +-
.../index/IndexOperationMessage.java | 65 ++
.../persistence/index/guice/IndexModule.java | 17 +-
.../index/impl/EsEntityIndexBatchImpl.java | 130 +--
.../index/impl/EsEntityIndexImpl.java | 88 +-
.../index/impl/EsIndexBufferConsumerImpl.java | 196 ++++
.../index/impl/EsIndexBufferProducerImpl.java | 57 ++
.../index/impl/CorePerformanceIT.java | 44 +-
.../impl/EntityConnectionIndexImplTest.java | 7 +-
.../persistence/index/impl/EntityIndexTest.java | 112 ++-
.../src/test/resources/log4j.properties | 4 +-
.../datagenerators/EntityDataGenerator.scala | 17 +-
.../datagenerators/FeederGenerator.scala | 24 +-
.../usergrid/scenarios/EntityScenarios.scala | 20 +-
.../org/apache/usergrid/settings/Settings.scala | 2 +
.../PostCustomEntitySimulation.scala | 27 +-
stack/mongo-emulator/pom.xml | 15 +-
stack/pom.xml | 71 +-
stack/rest/README.md | 27 +-
stack/rest/pom.xml | 185 +---
.../org/apache/usergrid/rest/IndexResource.java | 125 ++-
.../usergrid/rest/JobServiceBoostrap.java | 22 +-
.../org/apache/usergrid/rest/RootResource.java | 25 +-
.../apache/usergrid/rest/ShutdownListener.java | 75 ++
.../rest/applications/ApplicationResource.java | 72 +-
.../usergrid/rest/filters/MeteringFilter.java | 2 +
.../applications/ApplicationsResource.java | 29 +-
.../security/SecuredResourceFilterFactory.java | 2 +
.../security/shiro/filters/SecurityFilter.java | 3 +
.../usergrid/rest/test/PropertiesResource.java | 2 +-
.../resources/usergrid-rest-deploy-context.xml | 2 +-
stack/rest/src/main/webapp/WEB-INF/web.xml | 13 +-
.../apache/usergrid/rest/AbstractRestIT.java | 192 ++--
.../java/org/apache/usergrid/rest/BasicIT.java | 69 +-
.../usergrid/rest/DatabaseInitializer.java | 146 ---
.../java/org/apache/usergrid/rest/ITSetup.java | 57 +-
.../apache/usergrid/rest/IndexResourceIT.java | 3 +-
.../apache/usergrid/rest/PartialUpdateTest.java | 2 +-
.../apache/usergrid/rest/SystemResourceIT.java | 60 ++
.../org/apache/usergrid/rest/TomcatRuntime.java | 192 ++++
.../applications/ApplicationResourceIT.java | 941 +++++++++++-------
.../collection/BrowserCompatibilityTest.java | 68 +-
.../collection/CollectionsResourceIT.java | 252 +++--
.../collection/DuplicateNameIT.java | 63 +-
.../collection/groups/GroupResourceIT.java | 2 +-
.../users/ConnectionResourceTest.java | 272 +++--
.../collection/users/RetrieveUsersTest.java | 8 +-
.../applications/queries/GeoPagingTest.java | 44 +-
.../usergrid/rest/management/AdminUsersIT.java | 981 +++++++++----------
.../rest/management/ImportResourceIT.java | 5 +-
.../rest/management/OrganizationsIT.java | 2 +-
.../rest/management/RegistrationIT.java | 9 +-
.../organizations/AdminEmailEncodingIT.java | 161 +--
.../test/resource2point0/AbstractRestIT.java | 58 +-
.../rest/test/resource2point0/ClientSetup.java | 31 +-
.../rest/test/resource2point0/RestClient.java | 18 +
.../resource2point0/TestPropertiesResource.java | 47 +
.../endpoints/ApplicationsResource.java | 5 +
.../endpoints/CollectionEndpoint.java | 201 +++-
.../endpoints/DatabaseResource.java | 39 +
.../endpoints/EntityEndpoint.java | 2 -
.../endpoints/SetupResource.java | 46 +
.../endpoints/SystemResource.java | 36 +
.../endpoints/TokenResource.java | 32 +-
.../endpoints/mgmt/ApplicationResource.java | 6 +-
.../endpoints/mgmt/AuthorizeResource.java | 59 ++
.../endpoints/mgmt/ConfirmResource.java | 46 +
.../endpoints/mgmt/CredentialsResource.java | 55 ++
.../endpoints/mgmt/FeedResource.java | 48 +
.../endpoints/mgmt/ManagementResource.java | 12 +-
.../endpoints/mgmt/OrgResource.java | 96 +-
.../endpoints/mgmt/OrganizationResource.java | 44 +-
.../endpoints/mgmt/PasswordResource.java | 61 ++
.../endpoints/mgmt/ReactivateResource.java | 46 +
.../endpoints/mgmt/ResetResource.java | 42 +
.../endpoints/mgmt/TokenResource.java | 48 +-
.../endpoints/mgmt/UserResource.java | 80 ++
.../endpoints/mgmt/UsersResource.java | 13 +-
.../test/resource2point0/model/Credentials.java | 47 +
.../rest/test/resource2point0/model/Entity.java | 18 +-
.../rest/test/resource2point0/model/Token.java | 37 +-
stack/rest/src/test/resources/arquillian.xml | 16 +-
.../resources/usergrid-custom-test.properties | 14 +-
.../resources/usergrid-deployment.properties | 113 ---
.../resources/usergrid-rest-deploy-context.xml | 26 +
stack/services/pom.xml | 6 +
.../cassandra/ManagementServiceImpl.java | 7 +-
.../management/export/S3ExportImpl.java | 6 +-
.../management/importer/ImportServiceImpl.java | 5 +-
.../management/importer/S3ImportImpl.java | 28 +-
.../tokens/cassandra/TokenServiceImpl.java | 14 +-
.../services/assets/data/BinaryStore.java | 3 +-
.../services/assets/data/S3BinaryStore.java | 18 +-
.../usergrid/services/guice/ServiceModule.java | 2 -
.../notifications/NotificationsService.java | 4 +-
.../services/notifications/QueueJob.java | 6 +-
.../services/notifications/QueueListener.java | 17 +-
.../impl/ApplicationQueueManagerImpl.java | 2 +-
.../services/queues/ImportQueueListener.java | 7 +-
.../usergrid/services/queues/QueueListener.java | 7 +-
.../resources/usergrid-services-context.xml | 4 +-
.../org/apache/usergrid/ServiceITSetupImpl.java | 1 +
.../usergrid/management/OrganizationIT.java | 30 +-
.../management/export/ExportServiceIT.java | 183 ++--
.../importer/FileImportTrackerTest.java | 55 +-
.../management/importer/ImportCollectionIT.java | 78 +-
.../usergrid/management/importer/S3Upload.java | 18 +-
.../services/ApplicationsServiceIT.java | 8 +-
.../usergrid/services/ServiceInvocationIT.java | 7 +
.../apns/NotificationsServiceIT.java | 2 +-
.../gcm/NotificationsServiceIT.java | 4 +-
.../src/test/resources/log4j.properties | 4 +-
.../src/test/resources/project.properties | 1 +
.../resources/usergrid-custom-test.properties | 3 +
stack/test-utils/pom.xml | 11 -
.../usergrid/cassandra/ClearShiroSubject.java | 19 +-
161 files changed, 5466 insertions(+), 3243 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/08719b72/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index 9d0b03a,9aa36a0..e0d7daa
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@@ -15,26 -15,19 +15,18 @@@
*/
package org.apache.usergrid.corepersistence;
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheLoader;
-import com.google.common.cache.LoadingCache;
-import com.google.inject.Injector;
-import com.yammer.metrics.annotation.Metered;
-import static java.lang.String.CASE_INSENSITIVE_ORDER;
- import java.util.Arrays;
- import java.util.HashMap;
- import java.util.HashSet;
- import java.util.Iterator;
- import java.util.List;
- import java.util.Map;
- import java.util.Set;
- import java.util.TreeMap;
- import java.util.UUID;
+ import java.util.*;
import java.util.concurrent.atomic.AtomicBoolean;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.BeansException;
+import org.springframework.context.ApplicationContext;
+import org.springframework.context.ApplicationContextAware;
+
import org.apache.commons.lang.StringUtils;
-import org.apache.usergrid.corepersistence.rx.AllEntitiesInSystemObservable;
import org.apache.usergrid.corepersistence.util.CpNamingUtils;
import org.apache.usergrid.persistence.AbstractEntity;
import org.apache.usergrid.persistence.Entity;
@@@ -710,10 -763,15 +771,11 @@@ public class CpEntityManagerFactory imp
//explicitly invoke create index, we don't know if it exists or not in ES during a rebuild.
em.createIndex();
- Application app = em.getApplication();
+ em.reindex(po);
- Application app = em.getApplication();
- logger.info("\n\nRebuilt index for application {} id {}\n", app.getName(), appId );
+ em.reindex( po );
+
- if(app!=null) {
- logger.info("\n\nRebuilt index for application {} id {}\n", app.getName(), appId);
- }else{
- logger.info("\n\nDid not rebuild index for application id {}\n", appId);
- }
++ logger.info("\n\nRebuilt index for applicationId {} \n", appId );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/08719b72/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/08719b72/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/08719b72/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/guice/CommonModule.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/guice/CommonModule.java
index 97c0479,33321a4..5e61753
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/guice/CommonModule.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/guice/CommonModule.java
@@@ -19,7 -19,9 +19,10 @@@
package org.apache.usergrid.persistence.core.guice;
+ import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
+ import org.apache.usergrid.persistence.core.metrics.MetricsFactoryImpl;
+ import org.apache.usergrid.persistence.core.metrics.MetricsFig;
+import org.apache.usergrid.persistence.core.migration.data.*;
import org.safehaus.guicyfig.GuicyFigModule;
import org.apache.usergrid.persistence.core.astyanax.AstyanaxKeyspaceProvider;
@@@ -75,13 -81,16 +77,15 @@@ public class CommonModule extends Abstr
bind( DataMigrationManager.class ).to( DataMigrationManagerImpl.class );
+ bind( MetricsFactory.class ).to( MetricsFactoryImpl.class );
+ bind (MigrationInfoCache.class).to( MigrationInfoCacheImpl.class );
+ install(new GuicyFigModule(MetricsFig.class));
- //do multibindings for migrations
- Multibinder<DataMigration> dataMigrationMultibinder = Multibinder.newSetBinder( binder(), DataMigration.class );
-// dataMigrationMultibinder.addBinding();
-// dataMigrationManagerMultibinder.addBinding().to( DataMigrationManagerImpl.class );
-// migrationBinding.addBinding().to( Key.get( MigrationInfoSerialization.class ) );
+ //do multibindings for migrations
+ //create the empty multibinder so other plugins can use it
+ Multibinder.newSetBinder( binder(), MigrationPlugin.class);
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/08719b72/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/08719b72/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/08719b72/stack/pom.xml
----------------------------------------------------------------------
diff --cc stack/pom.xml
index 81e7bdd,53755b3..4c0dcd7
--- a/stack/pom.xml
+++ b/stack/pom.xml
@@@ -1535,8 -1513,14 +1513,14 @@@
<dependencies>
<dependency>
<groupId>org.apache.maven.surefire</groupId>
- <artifactId>surefire-junit47</artifactId>
+ <artifactId>surefire-junit4</artifactId>
<version>${surefire.plugin.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.maven.surfire</groupId>
+ <artifactId>common-junit3</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
</dependencies>
</plugin>
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/08719b72/stack/rest/src/main/java/org/apache/usergrid/rest/IndexResource.java
----------------------------------------------------------------------
diff --cc stack/rest/src/main/java/org/apache/usergrid/rest/IndexResource.java
index a0404e1,3269ec0..92d1e2b
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/IndexResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/IndexResource.java
@@@ -20,12 -20,9 +20,10 @@@
package org.apache.usergrid.rest;
+import com.google.common.base.Preconditions;
import com.sun.jersey.api.json.JSONWithPadding;
- import org.apache.usergrid.persistence.EntityManager;
import org.apache.usergrid.persistence.EntityManagerFactory;
import org.apache.usergrid.persistence.EntityRef;
- import org.apache.usergrid.persistence.index.EntityIndex;
import org.apache.usergrid.persistence.index.utils.UUIDUtils;
import org.apache.usergrid.rest.security.annotations.RequireSystemAccess;
import org.slf4j.Logger;
@@@ -183,11 -236,10 +237,12 @@@ public class IndexResource extends Abst
@POST
@Path( RootResource.APPLICATION_ID_PATH )
public JSONWithPadding addIndex(@Context UriInfo ui,
- @PathParam( "applicationId" ) final String applicationIdStr,
- Map<String, Object> config,
- @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback) throws Exception{
+ @PathParam( "applicationId" ) final String applicationIdStr,
+ Map<String, Object> config,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback) throws Exception{
+
+ Preconditions.checkNotNull(config,"Payload for config is null, please pass {replicas:int, shards:int} in body");
+
ApiResponse response = createApiResponse();
final UUID appId = UUIDUtils.tryExtractUUID(applicationIdStr);
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/08719b72/stack/services/src/test/java/org/apache/usergrid/management/OrganizationIT.java
----------------------------------------------------------------------
diff --cc stack/services/src/test/java/org/apache/usergrid/management/OrganizationIT.java
index 2babc60,bd96434..cc89122
--- a/stack/services/src/test/java/org/apache/usergrid/management/OrganizationIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/management/OrganizationIT.java
@@@ -90,9 -90,9 +90,9 @@@ public class OrganizationIT
OrganizationInfo organization2 = setup.getMgmtSvc().getOrganizationForApplication( applicationId );
assertNotNull( organization2 );
- assertEquals( "wrong organization name", orgName, organization2.getName() );
+ assertEquals( "wrong organization name", organization.getName(), organization2.getName() );
- boolean verified = setup.getMgmtSvc().verifyAdminUserPassword( user.getUuid(), "test" );
+ boolean verified = setup.getMgmtSvc().verifyAdminUserPassword( organization.getOwner().getUuid(), "test" );
assertTrue( verified );
setup.getMgmtSvc().activateOrganization( organization2 );
[20/50] [abbrv] incubator-usergrid git commit: Merge branch
'two-dot-o' into USERGRID-405
Posted by sf...@apache.org.
Merge branch 'two-dot-o' into USERGRID-405
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/e83caf28
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/e83caf28
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/e83caf28
Branch: refs/heads/USERGRID-480
Commit: e83caf2853b92c398d328c54de7665cb0fd56586
Parents: 49fe0ac 710a1a3
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Mar 4 18:55:13 2015 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Mar 4 18:55:13 2015 -0700
----------------------------------------------------------------------
.../java/org/apache/usergrid/persistence/index/query/Query.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
[02/50] [abbrv] incubator-usergrid git commit: Added new example
interfaces
Posted by sf...@apache.org.
Added new example interfaces
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/6652fe58
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/6652fe58
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/6652fe58
Branch: refs/heads/USERGRID-480
Commit: 6652fe581a91e52faee3508860a75795b320cbfd
Parents: 9589775
Author: Todd Nine <tn...@apigee.com>
Authored: Thu Feb 26 16:54:00 2015 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Thu Feb 26 16:54:00 2015 -0700
----------------------------------------------------------------------
.../migration/data/newimpls/DataMigration2.java | 71 ++++++++++++++++++++
.../data/newimpls/MigrationDataProvider.java | 43 ++++++++++++
.../data/newimpls/MigrationPlugin.java | 47 +++++++++++++
3 files changed, 161 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6652fe58/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/DataMigration2.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/DataMigration2.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/DataMigration2.java
new file mode 100644
index 0000000..f660c02
--- /dev/null
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/DataMigration2.java
@@ -0,0 +1,71 @@
+/*
+ *
+ * *
+ * * Licensed to the Apache Software Foundation (ASF) under one
+ * * or more contributor license agreements. See the NOTICE file
+ * * distributed with this work for additional information
+ * * regarding copyright ownership. The ASF licenses this file
+ * * to you under the Apache License, Version 2.0 (the
+ * * "License"); you may not use this file except in compliance
+ * * with the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing,
+ * * software distributed under the License is distributed on an
+ * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * * KIND, either express or implied. See the License for the
+ * * specific language governing permissions and limitations
+ * * under the License.
+ * *
+ *
+ */
+
+package org.apache.usergrid.persistence.core.migration.data.newimpls;
+
+
+/**
+ * Data migration. The internal version to migrate
+ *
+ * @param <T>
+ */
+public interface DataMigration2<T> {
+
+ /**
+ * Perform the migration, returning an observable with a single emitted value
+ * @param migrationDataProvider
+ */
+ public void migrate(MigrationDataProvider<T> migrationDataProvider, ProgressObserver observer);
+
+ /**
+ * Get the version of this migration. It should be unique within the scope of the plugin
+ * @return
+ */
+ public int version();
+
+
+ public interface ProgressObserver{
+ /**
+ * Mark the migration as failed
+ * @param migrationVersion The migration version running during the failure
+ * @param reason The reason to save
+ */
+ public void failed(final int migrationVersion, final String reason);
+
+ /**
+ * Mark the migration as failed with a stack trace
+ * @param migrationVersion The migration version running during the failure
+ * @param reason The error description to save
+ * @param throwable The error that happened
+ */
+ public void failed(final int migrationVersion, final String reason, final Throwable throwable);
+
+
+ /**
+ * Update the status of the migration with the message
+ *
+ * @param message The message to save for the status
+ */
+ public void update(final int migrationVersion, final String message);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6652fe58/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/MigrationDataProvider.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/MigrationDataProvider.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/MigrationDataProvider.java
new file mode 100644
index 0000000..ffe8d9e
--- /dev/null
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/MigrationDataProvider.java
@@ -0,0 +1,43 @@
+/*
+ *
+ * *
+ * * Licensed to the Apache Software Foundation (ASF) under one
+ * * or more contributor license agreements. See the NOTICE file
+ * * distributed with this work for additional information
+ * * regarding copyright ownership. The ASF licenses this file
+ * * to you under the Apache License, Version 2.0 (the
+ * * "License"); you may not use this file except in compliance
+ * * with the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing,
+ * * software distributed under the License is distributed on an
+ * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * * KIND, either express or implied. See the License for the
+ * * specific language governing permissions and limitations
+ * * under the License.
+ * *
+ *
+ */
+
+package org.apache.usergrid.persistence.core.migration.data.newimpls;
+
+
+import rx.Observable;
+
+
+/**
+ * An interface for data providers to implement. The migration must take the migrationdata provider as an argument
+ * @param <T>
+ */
+public interface MigrationDataProvider<T> {
+
+
+ /**
+ * Get data that can be used in the migration
+ * @return
+ */
+ public Observable<T> getData();
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6652fe58/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/MigrationPlugin.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/MigrationPlugin.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/MigrationPlugin.java
new file mode 100644
index 0000000..f04a035
--- /dev/null
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/newimpls/MigrationPlugin.java
@@ -0,0 +1,47 @@
+/*
+ *
+ * *
+ * * Licensed to the Apache Software Foundation (ASF) under one
+ * * or more contributor license agreements. See the NOTICE file
+ * * distributed with this work for additional information
+ * * regarding copyright ownership. The ASF licenses this file
+ * * to you under the Apache License, Version 2.0 (the
+ * * "License"); you may not use this file except in compliance
+ * * with the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing,
+ * * software distributed under the License is distributed on an
+ * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * * KIND, either express or implied. See the License for the
+ * * specific language governing permissions and limitations
+ * * under the License.
+ * *
+ *
+ */
+
+package org.apache.usergrid.persistence.core.migration.data.newimpls;
+
+
+import org.apache.usergrid.persistence.core.migration.data.DataMigration;
+
+
+/**
+ * A simple interface to return migration plugins. All versions within this migration plugin should have a name
+ */
+public interface MigrationPlugin {
+
+
+ /**
+ * Get the name of the plugin
+ * @return
+ */
+ public String getName();
+
+ /**
+ * Run any migrations that may need to be run in this plugin
+ */
+ public void run(DataMigration.ProgressObserver observer);
+
+}
[40/50] [abbrv] incubator-usergrid git commit: Fixes issue with
repair logic with incorrect assumptions of version ordering. Versions can
appear in any order during arbitrary order by statements.
Posted by sf...@apache.org.
Fixes issue with repair logic with incorrect assumptions of version ordering. Versions can appear in any order during arbitrary order by statements.
Also fixes overflow logic during repair when you may be more results than anticipated.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/407a4f55
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/407a4f55
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/407a4f55
Branch: refs/heads/USERGRID-480
Commit: 407a4f55eff3adb01f69e7e45f943918e60dd240
Parents: 0334119
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Mar 18 15:53:43 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Mar 18 15:53:43 2015 -0600
----------------------------------------------------------------------
.../corepersistence/CpRelationManager.java | 7 ++-
.../results/FilteringLoader.java | 50 +++++++++++++-------
2 files changed, 38 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/407a4f55/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index ee81a67..2eeee28 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -987,7 +987,12 @@ public class CpRelationManager implements RelationManager {
if ( crs.isEmpty() || !crs.hasCursor() ) { // no results, no cursor, can't get more
satisfied = true;
}
- else if ( results.size() == originalLimit ) { // got what we need
+
+ /**
+ * In an edge case where we delete stale entities, we could potentially get more results than expected. This will only occur once during the repair phase.
+ * We need to ensure that we short circuit if we overflow the requested limit during a repair.
+ */
+ else if ( results.size() >= originalLimit ) { // got what we need
satisfied = true;
}
else if ( crs.hasCursor() ) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/407a4f55/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java
index 2cd9fdc..b571f21 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java
@@ -143,30 +143,44 @@ public class FilteringLoader implements ResultsLoader {
final UUID currentVersion = currentCandidate.getVersion();
+
+ final CandidateResult toRemove;
+ final CandidateResult toKeep;
+
+ //current is newer than previous. Remove previous and keep current
+ if(UUIDComparator.staticCompare( currentVersion, previousMaxVersion ) > 0 ){
+ toRemove = previousMax;
+ toKeep = currentCandidate;
+ }
+ //previously seen value is newer than current. Remove the current and keep the previously seen value
+ else{
+ toRemove = currentCandidate;
+ toKeep = previousMax;
+ }
+
//this is a newer version, we know we already have a stale entity, add it to be cleaned up
- if ( UUIDComparator.staticCompare( currentVersion, previousMaxVersion ) > 0 ) {
- //de-index it
- logger.warn( "Stale version of Entity uuid:{} type:{}, stale v:{}, latest v:{}",
- new Object[] {
- entityId.getUuid(),
- entityId.getType(),
- previousMaxVersion,
- currentVersion } );
- //deindex this document, and remove the previous maxVersion
- //we have to deindex this from our ownerId, since this is what gave us the reference
- indexBatch.deindex( indexScope, previousMax );
- groupedByScopes.remove( collectionType, previousMax );
+ //de-index it
+ logger.warn( "Stale version of Entity uuid:{} type:{}, stale v:{}, latest v:{}",
+ new Object[] {
+ entityId.getUuid(),
+ entityId.getType(),
+ toRemove.getVersion(),
+ toKeep.getVersion() } );
+ //deindex this document, and remove the previous maxVersion
+ //we have to deindex this from our ownerId, since this is what gave us the reference
+ indexBatch.deindex( indexScope, toRemove );
+ groupedByScopes.remove( collectionType, toRemove );
- //TODO, fire the entity repair cleanup task here instead of de-indexing
- //replace the value with a more current version
- maxCandidateMapping.put( entityId, currentCandidate );
- orderIndex.put( entityId, i );
- groupedByScopes.put( collectionType, currentCandidate );
- }
+ //TODO, fire the entity repair cleanup task here instead of de-indexing
+
+ //replace the value with a more current version
+ maxCandidateMapping.put( entityId, toKeep );
+ orderIndex.put( entityId, i );
+ groupedByScopes.put( collectionType, toKeep );
}
[27/50] [abbrv] incubator-usergrid git commit: Renames listener to
complete.
Posted by sf...@apache.org.
Renames listener to complete.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/dc87edc1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/dc87edc1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/dc87edc1
Branch: refs/heads/USERGRID-480
Commit: dc87edc13e3c1241dc9ab6bed56b11f8a953e9b0
Parents: 275e74a
Author: Todd Nine <tn...@apigee.com>
Authored: Fri Mar 6 11:33:04 2015 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Fri Mar 6 11:33:04 2015 -0700
----------------------------------------------------------------------
.../corepersistence/migration/MigrationModuleVersionPlugin.java | 5 +----
.../core/migration/data/AbstractMigrationPlugin.java | 2 +-
.../core/migration/data/DataMigrationManagerImpl.java | 2 +-
.../persistence/core/migration/data/ProgressObserver.java | 2 +-
.../persistence/core/migration/data/TestProgressObserver.java | 2 +-
5 files changed, 5 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/dc87edc1/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/MigrationModuleVersionPlugin.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/MigrationModuleVersionPlugin.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/MigrationModuleVersionPlugin.java
index e548410..00e7f11 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/MigrationModuleVersionPlugin.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/MigrationModuleVersionPlugin.java
@@ -22,10 +22,7 @@ package org.apache.usergrid.corepersistence.migration;
import org.apache.usergrid.persistence.collection.serialization.impl.MvccEntitySerializationStrategyV2Impl;
import org.apache.usergrid.persistence.collection.serialization.impl.migration.CollectionMigrationPlugin;
-import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
import org.apache.usergrid.persistence.core.migration.data.MigrationInfoSerialization;
-import org.apache.usergrid.persistence.core.migration.data.DataMigration;
-import org.apache.usergrid.persistence.core.migration.data.MigrationDataProvider;
import org.apache.usergrid.persistence.core.migration.data.MigrationPlugin;
import org.apache.usergrid.persistence.core.migration.data.PluginPhase;
import org.apache.usergrid.persistence.core.migration.data.ProgressObserver;
@@ -110,7 +107,7 @@ public class MigrationModuleVersionPlugin implements MigrationPlugin{
//save the version
migrationInfoSerialization.setVersion( NAME, getMaxVersion() );
- observer.stop();
+ observer.complete();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/dc87edc1/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/AbstractMigrationPlugin.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/AbstractMigrationPlugin.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/AbstractMigrationPlugin.java
index 1d351d1..0cd26d4 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/AbstractMigrationPlugin.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/AbstractMigrationPlugin.java
@@ -118,7 +118,7 @@ public abstract class AbstractMigrationPlugin<T> implements MigrationPlugin {
//write the version
migrationInfoSerialization.setVersion( getName(), newSystemVersion );
- po.stop();
+ po.complete();
//signal we've run a migration and return
return true;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/dc87edc1/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManagerImpl.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManagerImpl.java
index 3f763a2..fe9e741 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManagerImpl.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManagerImpl.java
@@ -194,7 +194,7 @@ public class DataMigrationManagerImpl implements DataMigrationManager {
@Override
- public void stop() {
+ public void complete() {
migrationInfoSerialization.setStatusCode( pluginName, StatusCode.COMPLETE.status );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/dc87edc1/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/ProgressObserver.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/ProgressObserver.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/ProgressObserver.java
index 50abc6f..43987eb 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/ProgressObserver.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/ProgressObserver.java
@@ -34,7 +34,7 @@ public interface ProgressObserver {
/**
* Signal the process has stopped
*/
- public void stop();
+ public void complete();
/**
* Mark the migration as failed
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/dc87edc1/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/TestProgressObserver.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/TestProgressObserver.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/TestProgressObserver.java
index 1612779..7f59151 100644
--- a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/TestProgressObserver.java
+++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/TestProgressObserver.java
@@ -41,7 +41,7 @@ public class TestProgressObserver implements ProgressObserver {
@Override
- public void stop() {
+ public void complete() {
started = false;
}
[26/50] [abbrv] incubator-usergrid git commit: Refactored plugins to
allow a "bootstrap" system plugin.
Posted by sf...@apache.org.
Refactored plugins to allow a "bootstrap" system plugin.
Also added phase configuration for plugins, so if they require root configuration, it can be implemented.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/275e74a5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/275e74a5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/275e74a5
Branch: refs/heads/USERGRID-480
Commit: 275e74a5b83b36ee0b75f5f2fc6d6f21faeb7e82
Parents: 4ee788d
Author: Todd Nine <tn...@apigee.com>
Authored: Thu Mar 5 21:03:10 2015 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Thu Mar 5 21:03:10 2015 -0700
----------------------------------------------------------------------
stack/core/pom.xml | 10 +
.../usergrid/corepersistence/CoreModule.java | 10 +-
.../migration/CoreDataVersions.java | 3 +-
.../migration/CoreMigrationPlugin.java | 5 +
.../migration/EntityTypeMappingMigration.java | 2 +-
.../migration/MigrationModuleVersion.java | 112 --------
.../migration/MigrationModuleVersionPlugin.java | 136 ++++++++++
.../migration/MigrationSystemVersions.java | 43 +++
.../MigrationModuleVersionPluginTest.java | 259 +++++++++++++++++++
.../migration/CollectionMigrationPlugin.java | 5 +
.../data/DataMigrationManagerImpl.java | 49 +++-
.../core/migration/data/MigrationPlugin.java | 7 +-
.../core/migration/data/PluginPhase.java | 42 +++
.../data/DataMigrationManagerImplTest.java | 119 ++++++---
.../impl/migration/GraphMigrationPlugin.java | 7 +
15 files changed, 648 insertions(+), 161 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/275e74a5/stack/core/pom.xml
----------------------------------------------------------------------
diff --git a/stack/core/pom.xml b/stack/core/pom.xml
index d30bedb..23a0618 100644
--- a/stack/core/pom.xml
+++ b/stack/core/pom.xml
@@ -423,6 +423,15 @@
<scope>test</scope>
</dependency>
+
+ <dependency>
+ <groupId>org.apache.usergrid</groupId>
+ <artifactId>common</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+
<!-- Core Persistence deps -->
<dependency>
<groupId>org.apache.usergrid</groupId>
@@ -441,6 +450,7 @@
</exclusions>
</dependency>
+
<dependency>
<groupId>org.apache.usergrid</groupId>
<artifactId>queryindex</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/275e74a5/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
index 62be6cf..cf927ce 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
@@ -24,7 +24,7 @@ import org.apache.usergrid.corepersistence.events.EntityVersionDeletedHandler;
import org.apache.usergrid.corepersistence.migration.CoreMigration;
import org.apache.usergrid.corepersistence.migration.CoreMigrationPlugin;
import org.apache.usergrid.corepersistence.migration.EntityTypeMappingMigration;
-import org.apache.usergrid.corepersistence.migration.MigrationModuleVersion;
+import org.apache.usergrid.corepersistence.migration.MigrationModuleVersionPlugin;
import org.apache.usergrid.corepersistence.rx.impl.AllEntitiesInSystemImpl;
import org.apache.usergrid.corepersistence.rx.impl.AllNodesInGraphImpl;
import org.apache.usergrid.corepersistence.rx.impl.AllApplicationsObservableImpl;
@@ -34,7 +34,6 @@ import org.apache.usergrid.persistence.collection.event.EntityVersionCreated;
import org.apache.usergrid.persistence.collection.event.EntityVersionDeleted;
import org.apache.usergrid.persistence.collection.guice.CollectionModule;
import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
-import org.apache.usergrid.persistence.collection.serialization.impl.migration.MvccEntityDataMigrationImpl;
import org.apache.usergrid.persistence.core.guice.CommonModule;
import org.apache.usergrid.persistence.core.migration.data.DataMigration;
import org.apache.usergrid.persistence.core.migration.data.MigrationDataProvider;
@@ -128,13 +127,14 @@ public class CoreModule extends AbstractModule {
dataMigrationMultibinder.addBinding().to( EntityTypeMappingMigration.class );
- dataMigrationMultibinder.addBinding().to( MigrationModuleVersion.class );
//wire up the collection migration plugin
- Multibinder.newSetBinder( binder(), MigrationPlugin.class ).addBinding().to( CoreMigrationPlugin.class );
+ final Multibinder<MigrationPlugin> plugins = Multibinder.newSetBinder( binder(), MigrationPlugin.class );
+ plugins.addBinding().to( CoreMigrationPlugin.class );
+ plugins.addBinding().to(MigrationModuleVersionPlugin.class );
- bind(AllApplicationsObservable.class).to(AllApplicationsObservableImpl.class);
+ bind( AllApplicationsObservable.class).to(AllApplicationsObservableImpl.class);
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/275e74a5/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/CoreDataVersions.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/CoreDataVersions.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/CoreDataVersions.java
index 58ba6a3..94ff88c 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/CoreDataVersions.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/CoreDataVersions.java
@@ -28,8 +28,7 @@ public enum CoreDataVersions {
//up to date so that our new migration module can proceed.
INITIAL(0),
- MIGRATION_VERSION_FIX(1),
- ID_MAP_FIX(2);
+ ID_MAP_FIX(1);
private final int version;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/275e74a5/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/CoreMigrationPlugin.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/CoreMigrationPlugin.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/CoreMigrationPlugin.java
index 91c1570..58233e0 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/CoreMigrationPlugin.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/CoreMigrationPlugin.java
@@ -27,6 +27,7 @@ import org.apache.usergrid.persistence.core.migration.data.MigrationInfoSerializ
import org.apache.usergrid.persistence.core.migration.data.AbstractMigrationPlugin;
import org.apache.usergrid.persistence.core.migration.data.DataMigration;
import org.apache.usergrid.persistence.core.migration.data.MigrationDataProvider;
+import org.apache.usergrid.persistence.core.migration.data.PluginPhase;
import com.google.inject.Inject;
import com.google.inject.Singleton;
@@ -56,4 +57,8 @@ public class CoreMigrationPlugin extends AbstractMigrationPlugin<EntityIdScope>
}
+ @Override
+ public PluginPhase getPhase() {
+ return PluginPhase.MIGRATE;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/275e74a5/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/EntityTypeMappingMigration.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/EntityTypeMappingMigration.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/EntityTypeMappingMigration.java
index 1a5ef13..ad50c18 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/EntityTypeMappingMigration.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/EntityTypeMappingMigration.java
@@ -99,7 +99,7 @@ public class EntityTypeMappingMigration implements DataMigration<EntityIdScope>
@Override
public boolean supports( final int currentVersion ) {
//we move from the migration version fix to the current version
- return CoreDataVersions.MIGRATION_VERSION_FIX.getVersion() == currentVersion;
+ return CoreDataVersions.INITIAL.getVersion() == currentVersion;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/275e74a5/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/MigrationModuleVersion.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/MigrationModuleVersion.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/MigrationModuleVersion.java
deleted file mode 100644
index 9ddc219..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/MigrationModuleVersion.java
+++ /dev/null
@@ -1,112 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.migration;
-
-
-import org.apache.usergrid.persistence.collection.serialization.impl.MvccEntitySerializationStrategyV2Impl;
-import org.apache.usergrid.persistence.collection.serialization.impl.migration.CollectionMigrationPlugin;
-import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
-import org.apache.usergrid.persistence.core.migration.data.MigrationInfoSerialization;
-import org.apache.usergrid.persistence.core.migration.data.DataMigration;
-import org.apache.usergrid.persistence.core.migration.data.MigrationDataProvider;
-import org.apache.usergrid.persistence.core.migration.data.ProgressObserver;
-import org.apache.usergrid.persistence.graph.serialization.impl.EdgeMetadataSerializationV2Impl;
-import org.apache.usergrid.persistence.graph.serialization.impl.migration.GraphMigrationPlugin;
-
-import com.google.inject.Inject;
-
-
-/**
- * Migration to set our module versions now that we've refactor for sub modules Keeps the EntityIdScope because it won't
- * subscribe to the data provider.
- */
-public class MigrationModuleVersion implements DataMigration<EntityIdScope> {
-
-
- /**
- * The migration from 0 -> 1 that re-writes all the entity id's into the map module
- */
- private static final int ID_MIGRATION = 1;
-
- /**
- * The migration from 1-> 2 that shards our edge meta data
- */
- private static final int EDGE_SHARD_MIGRATION = 2;
-
- /**
- * The migration from 2-> 3 that fixed the short truncation bug
- */
- private static final int ENTITY_V2_MIGRATION = 3;
-
- private final MigrationInfoSerialization migrationInfoSerialization;
-
- private final MvccEntitySerializationStrategyV2Impl serializationStrategyV2;
-
- private final EdgeMetadataSerializationV2Impl edgeMetadataSerializationV2;
-
-
- @Inject
- public MigrationModuleVersion( final MigrationInfoSerialization migrationInfoSerialization,
- final MvccEntitySerializationStrategyV2Impl serializationStrategyV2,
- final EdgeMetadataSerializationV2Impl edgeMetadataSerializationV2 ) {
- this.migrationInfoSerialization = migrationInfoSerialization;
- this.serializationStrategyV2 = serializationStrategyV2;
- this.edgeMetadataSerializationV2 = edgeMetadataSerializationV2;
- }
-
-
- @Override
- public int migrate( final int currentVersion, final MigrationDataProvider<EntityIdScope> migrationDataProvider,
- final ProgressObserver observer ) {
-
- //we ignore our current version, since it will always be 0
- final int legacyVersion = migrationInfoSerialization.getSystemVersion();
-
- //now we store versions for each of our modules
-
- switch ( legacyVersion ) {
- case ID_MIGRATION:
- //no op, we need to migration everything in all modules
- break;
- //we need to set the version of the entity data, and our edge shard migration. The fall through (no break) is deliberate
- case ENTITY_V2_MIGRATION:
- migrationInfoSerialization.setVersion( CollectionMigrationPlugin.PLUGIN_NAME, serializationStrategyV2.getImplementationVersion() );
- case EDGE_SHARD_MIGRATION:
- //set our shard migration to the migrated version
- migrationInfoSerialization.setVersion( GraphMigrationPlugin.PLUGIN_NAME, edgeMetadataSerializationV2.getImplementationVersion() );
- break;
- }
-
- return CoreDataVersions.MIGRATION_VERSION_FIX.getVersion();
- }
-
-
- @Override
- public boolean supports( final int currentVersion ) {
- //we move from the migration version fix to the current version
- return CoreDataVersions.INITIAL.getVersion() == currentVersion;
- }
-
-
- @Override
- public int getMaxVersion() {
- return CoreDataVersions.MIGRATION_VERSION_FIX.getVersion();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/275e74a5/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/MigrationModuleVersionPlugin.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/MigrationModuleVersionPlugin.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/MigrationModuleVersionPlugin.java
new file mode 100644
index 0000000..e548410
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/MigrationModuleVersionPlugin.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.migration;
+
+
+import org.apache.usergrid.persistence.collection.serialization.impl.MvccEntitySerializationStrategyV2Impl;
+import org.apache.usergrid.persistence.collection.serialization.impl.migration.CollectionMigrationPlugin;
+import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
+import org.apache.usergrid.persistence.core.migration.data.MigrationInfoSerialization;
+import org.apache.usergrid.persistence.core.migration.data.DataMigration;
+import org.apache.usergrid.persistence.core.migration.data.MigrationDataProvider;
+import org.apache.usergrid.persistence.core.migration.data.MigrationPlugin;
+import org.apache.usergrid.persistence.core.migration.data.PluginPhase;
+import org.apache.usergrid.persistence.core.migration.data.ProgressObserver;
+import org.apache.usergrid.persistence.graph.serialization.impl.EdgeMetadataSerializationV2Impl;
+import org.apache.usergrid.persistence.graph.serialization.impl.migration.GraphMigrationPlugin;
+
+import com.google.inject.Inject;
+
+
+/**
+ * Migration to set our module versions now that we've refactor for sub modules Keeps the EntityIdScope because it won't
+ * subscribe to the data provider.
+ */
+public class MigrationModuleVersionPlugin implements MigrationPlugin{
+
+ public static final String NAME = "migration-system";
+
+ private static final int INITIAL = 0;
+ /**
+ * The migration from 0 -> 1 that re-writes all the entity id's into the map module
+ */
+ private static final int ID_MIGRATION = 1;
+
+ /**
+ * The migration from 1-> 2 that shards our edge meta data
+ */
+ private static final int EDGE_SHARD_MIGRATION = 2;
+
+ /**
+ * The migration from 2-> 3 that fixed the short truncation bug
+ */
+ private static final int ENTITY_V2_MIGRATION = 3;
+
+
+ /**
+ * Get versions directly from impls so we know they're accurate
+ */
+ private final MigrationInfoSerialization migrationInfoSerialization;
+
+ private final MvccEntitySerializationStrategyV2Impl serializationStrategyV2;
+
+ private final EdgeMetadataSerializationV2Impl edgeMetadataSerializationV2;
+
+
+ @Inject
+ public MigrationModuleVersionPlugin( final MigrationInfoSerialization migrationInfoSerialization,
+ final MvccEntitySerializationStrategyV2Impl serializationStrategyV2,
+ final EdgeMetadataSerializationV2Impl edgeMetadataSerializationV2 ) {
+ this.migrationInfoSerialization = migrationInfoSerialization;
+ this.serializationStrategyV2 = serializationStrategyV2;
+ this.edgeMetadataSerializationV2 = edgeMetadataSerializationV2;
+ }
+
+ @Override
+ public void run( final ProgressObserver observer ) {
+
+
+ observer.start();
+
+ //we ignore our current version, since it will always be 0
+ final int legacyVersion = migrationInfoSerialization.getSystemVersion();
+
+
+
+ //now we store versions for each of our modules
+
+ switch ( legacyVersion ) {
+ //we need to set the version of the entity data, and our edge shard migration. The fall through (no break) is deliberate
+ //if it's initial, set both
+ case INITIAL:
+ //if it's entity v2, set all, it's current
+ case ENTITY_V2_MIGRATION:
+ migrationInfoSerialization.setVersion( CollectionMigrationPlugin.PLUGIN_NAME, serializationStrategyV2.getImplementationVersion() );
+ //if it's edge shard, we need to run the v2 migration
+ case EDGE_SHARD_MIGRATION:
+ //set our shard migration to the migrated version
+ migrationInfoSerialization.setVersion( GraphMigrationPlugin.PLUGIN_NAME, edgeMetadataSerializationV2.getImplementationVersion() );
+ case ID_MIGRATION:
+ migrationInfoSerialization.setVersion( CoreMigrationPlugin.PLUGIN_NAME, CoreDataVersions.ID_MAP_FIX.getVersion() );
+ }
+
+ //save the version
+ migrationInfoSerialization.setVersion( NAME, getMaxVersion() );
+
+ observer.stop();
+ }
+
+
+
+ @Override
+ public String getName() {
+ return NAME;
+ }
+
+
+
+
+ @Override
+ public int getMaxVersion() {
+ return MigrationSystemVersions.LEGACY_ID_MAPPED.getVersion();
+ }
+
+
+ @Override
+ public PluginPhase getPhase() {
+ return PluginPhase.BOOTSTRAP;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/275e74a5/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/MigrationSystemVersions.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/MigrationSystemVersions.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/MigrationSystemVersions.java
new file mode 100644
index 0000000..8b96541
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/MigrationSystemVersions.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.migration;
+
+
+/**
+ * Versions of data as they exist across our system
+ */
+public enum MigrationSystemVersions {
+ //even though this didn't really come first in time, we need to run this first in order to bring our system
+ //up to date so that our new migration module can proceed.
+
+ INITIAL(0),
+ LEGACY_ID_MAPPED(1);
+
+
+ private final int version;
+
+
+ private MigrationSystemVersions( final int version ) {this.version = version;}
+
+
+ public int getVersion() {
+ return version;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/275e74a5/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/MigrationModuleVersionPluginTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/MigrationModuleVersionPluginTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/MigrationModuleVersionPluginTest.java
new file mode 100644
index 0000000..657bd16
--- /dev/null
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/MigrationModuleVersionPluginTest.java
@@ -0,0 +1,259 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.migration;
+
+
+import org.junit.Test;
+
+import org.apache.usergrid.persistence.collection.serialization.MvccEntitySerializationStrategy;
+import org.apache.usergrid.persistence.collection.serialization.impl.MvccEntitySerializationStrategyV2Impl;
+import org.apache.usergrid.persistence.collection.serialization.impl.migration.CollectionMigrationPlugin;
+import org.apache.usergrid.persistence.core.migration.data.MigrationInfoSerialization;
+import org.apache.usergrid.persistence.core.migration.data.TestProgressObserver;
+import org.apache.usergrid.persistence.graph.serialization.EdgeMetadataSerialization;
+import org.apache.usergrid.persistence.graph.serialization.impl.EdgeMetadataSerializationV2Impl;
+import org.apache.usergrid.persistence.graph.serialization.impl.migration.GraphMigrationPlugin;
+
+import com.google.inject.Inject;
+
+import static org.junit.Assert.*;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+
+/**
+ * Tests the system sets itself up as wxpected
+ */
+public class MigrationModuleVersionPluginTest {
+
+
+ @Test
+ public void testNewSystem(){
+
+ //mock up an initial system state
+ final int systemState = 0;
+
+ final MigrationInfoSerialization migrationInfoSerialization = mock(MigrationInfoSerialization.class);
+
+ when(migrationInfoSerialization.getSystemVersion()).thenReturn( systemState );
+
+
+
+
+ final int mvccVersion = 2;
+
+ final MvccEntitySerializationStrategyV2Impl serializationStrategyV2 = mock(MvccEntitySerializationStrategyV2Impl.class);
+ when(serializationStrategyV2.getImplementationVersion()).thenReturn( mvccVersion );
+
+
+ final int edgeMetaVersion = 1;
+
+ final EdgeMetadataSerializationV2Impl edgeMetadataSerializationV2 = mock(EdgeMetadataSerializationV2Impl.class);
+ when(edgeMetadataSerializationV2.getImplementationVersion()).thenReturn(edgeMetaVersion);
+
+
+ final MigrationModuleVersionPlugin plugin = new MigrationModuleVersionPlugin(migrationInfoSerialization, serializationStrategyV2, edgeMetadataSerializationV2 );
+
+ final TestProgressObserver testProgressObserver = new TestProgressObserver();
+
+ plugin.run( testProgressObserver );
+
+
+
+ //first version that should be set
+ verify(migrationInfoSerialization).setVersion( CoreMigrationPlugin.PLUGIN_NAME, CoreDataVersions.ID_MAP_FIX.getVersion() );
+
+ //second version that should be set
+
+ verify(migrationInfoSerialization).setVersion( GraphMigrationPlugin.PLUGIN_NAME, edgeMetaVersion );
+
+ //last version that should be set
+ verify(migrationInfoSerialization).setVersion( CollectionMigrationPlugin.PLUGIN_NAME, mvccVersion );
+
+
+ //set this plugin as run
+ verify(migrationInfoSerialization).setVersion( MigrationModuleVersionPlugin.NAME, MigrationSystemVersions.LEGACY_ID_MAPPED.getVersion() );
+
+
+
+ }
+
+
+
+ @Test
+ public void testIdMapping(){
+
+ //mock up an initial system state
+ final int systemState = 1;
+
+ final MigrationInfoSerialization migrationInfoSerialization = mock(MigrationInfoSerialization.class);
+
+ when(migrationInfoSerialization.getSystemVersion()).thenReturn( systemState );
+
+
+
+
+ final int mvccVersion = 2;
+
+ final MvccEntitySerializationStrategyV2Impl serializationStrategyV2 = mock(MvccEntitySerializationStrategyV2Impl.class);
+ when(serializationStrategyV2.getImplementationVersion()).thenReturn( mvccVersion );
+
+
+ final int edgeMetaVersion = 1;
+
+ final EdgeMetadataSerializationV2Impl edgeMetadataSerializationV2 = mock(EdgeMetadataSerializationV2Impl.class);
+ when(edgeMetadataSerializationV2.getImplementationVersion()).thenReturn(edgeMetaVersion);
+
+
+ final MigrationModuleVersionPlugin plugin = new MigrationModuleVersionPlugin(migrationInfoSerialization, serializationStrategyV2, edgeMetadataSerializationV2 );
+
+ final TestProgressObserver testProgressObserver = new TestProgressObserver();
+
+ plugin.run( testProgressObserver );
+
+
+ //first version that should be set
+ verify(migrationInfoSerialization).setVersion( CoreMigrationPlugin.PLUGIN_NAME, CoreDataVersions.ID_MAP_FIX.getVersion() );
+
+ //second version that should be set
+
+ verify(migrationInfoSerialization, never()).setVersion( GraphMigrationPlugin.PLUGIN_NAME, edgeMetaVersion );
+
+ //last version that should be set
+ verify(migrationInfoSerialization, never()).setVersion( CollectionMigrationPlugin.PLUGIN_NAME, mvccVersion );
+
+
+ //set this plugin as run
+ verify(migrationInfoSerialization).setVersion( MigrationModuleVersionPlugin.NAME, MigrationSystemVersions.LEGACY_ID_MAPPED.getVersion() );
+
+
+
+
+ }
+
+
+ @Test
+ public void testEdgeMigration(){
+
+ //mock up an initial system state
+ final int systemState = 2;
+
+ final MigrationInfoSerialization migrationInfoSerialization = mock(MigrationInfoSerialization.class);
+
+ when(migrationInfoSerialization.getSystemVersion()).thenReturn( systemState );
+
+
+
+
+ final int mvccVersion = 2;
+
+ final MvccEntitySerializationStrategyV2Impl serializationStrategyV2 = mock(MvccEntitySerializationStrategyV2Impl.class);
+ when(serializationStrategyV2.getImplementationVersion()).thenReturn( mvccVersion );
+
+
+ final int edgeMetaVersion = 1;
+
+ final EdgeMetadataSerializationV2Impl edgeMetadataSerializationV2 = mock(EdgeMetadataSerializationV2Impl.class);
+ when(edgeMetadataSerializationV2.getImplementationVersion()).thenReturn(edgeMetaVersion);
+
+
+ final MigrationModuleVersionPlugin plugin = new MigrationModuleVersionPlugin(migrationInfoSerialization, serializationStrategyV2, edgeMetadataSerializationV2 );
+
+ final TestProgressObserver testProgressObserver = new TestProgressObserver();
+
+ plugin.run( testProgressObserver );
+
+
+ //first version that should be set
+ verify(migrationInfoSerialization).setVersion( CoreMigrationPlugin.PLUGIN_NAME, CoreDataVersions.ID_MAP_FIX.getVersion() );
+
+ //second version that should be set
+
+ verify(migrationInfoSerialization).setVersion( GraphMigrationPlugin.PLUGIN_NAME, edgeMetaVersion );
+
+ //last version that should be set
+ verify(migrationInfoSerialization, never()).setVersion( CollectionMigrationPlugin.PLUGIN_NAME, mvccVersion );
+
+
+ //set this plugin as run
+ verify(migrationInfoSerialization).setVersion( MigrationModuleVersionPlugin.NAME, MigrationSystemVersions.LEGACY_ID_MAPPED.getVersion() );
+
+
+
+
+ }
+
+
+
+
+ @Test
+ public void testEntityV2Migration(){
+
+ //mock up an initial system state
+ final int systemState = 3;
+
+ final MigrationInfoSerialization migrationInfoSerialization = mock(MigrationInfoSerialization.class);
+
+ when(migrationInfoSerialization.getSystemVersion()).thenReturn( systemState );
+
+
+
+
+ final int mvccVersion = 2;
+
+ final MvccEntitySerializationStrategyV2Impl serializationStrategyV2 = mock(MvccEntitySerializationStrategyV2Impl.class);
+ when(serializationStrategyV2.getImplementationVersion()).thenReturn( mvccVersion );
+
+
+ final int edgeMetaVersion = 1;
+
+ final EdgeMetadataSerializationV2Impl edgeMetadataSerializationV2 = mock(EdgeMetadataSerializationV2Impl.class);
+ when(edgeMetadataSerializationV2.getImplementationVersion()).thenReturn(edgeMetaVersion);
+
+
+ final MigrationModuleVersionPlugin plugin = new MigrationModuleVersionPlugin(migrationInfoSerialization, serializationStrategyV2, edgeMetadataSerializationV2 );
+
+ final TestProgressObserver testProgressObserver = new TestProgressObserver();
+
+ plugin.run( testProgressObserver );
+
+
+ //first version that should be set
+ verify(migrationInfoSerialization).setVersion( CoreMigrationPlugin.PLUGIN_NAME, CoreDataVersions.ID_MAP_FIX.getVersion() );
+
+ //second version that should be set
+
+ verify(migrationInfoSerialization).setVersion( GraphMigrationPlugin.PLUGIN_NAME, edgeMetaVersion );
+
+ //last version that should be set
+ verify(migrationInfoSerialization).setVersion( CollectionMigrationPlugin.PLUGIN_NAME, mvccVersion );
+
+
+ //set this plugin as run
+ verify(migrationInfoSerialization).setVersion( MigrationModuleVersionPlugin.NAME, MigrationSystemVersions.LEGACY_ID_MAPPED.getVersion() );
+
+
+
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/275e74a5/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/CollectionMigrationPlugin.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/CollectionMigrationPlugin.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/CollectionMigrationPlugin.java
index 55e42ff..3f83c66 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/CollectionMigrationPlugin.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/CollectionMigrationPlugin.java
@@ -30,6 +30,7 @@ import org.apache.usergrid.persistence.core.migration.data.MigrationInfoSerializ
import org.apache.usergrid.persistence.core.migration.data.AbstractMigrationPlugin;
import org.apache.usergrid.persistence.core.migration.data.DataMigration;
import org.apache.usergrid.persistence.core.migration.data.MigrationDataProvider;
+import org.apache.usergrid.persistence.core.migration.data.PluginPhase;
import com.google.inject.Inject;
import com.google.inject.Singleton;
@@ -60,4 +61,8 @@ public class CollectionMigrationPlugin extends AbstractMigrationPlugin<EntityIdS
}
+ @Override
+ public PluginPhase getPhase() {
+ return PluginPhase.MIGRATE;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/275e74a5/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManagerImpl.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManagerImpl.java
index 7b97e8e..3f763a2 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManagerImpl.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManagerImpl.java
@@ -21,7 +21,11 @@ package org.apache.usergrid.persistence.core.migration.data;
import java.io.PrintWriter;
import java.io.StringWriter;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -40,7 +44,9 @@ public class DataMigrationManagerImpl implements DataMigrationManager {
private static final Logger LOG = LoggerFactory.getLogger( DataMigrationManagerImpl.class );
- private final Map<String, MigrationPlugin> migrationPlugins = new HashMap<>();
+ private final Map<String, MigrationPlugin> migrationPlugins;
+
+ private final List<MigrationPlugin> executionOrder;
private final MigrationInfoSerialization migrationInfoSerialization;
@@ -50,12 +56,17 @@ public class DataMigrationManagerImpl implements DataMigrationManager {
public DataMigrationManagerImpl( final Set<MigrationPlugin> plugins,
final MigrationInfoSerialization migrationInfoSerialization ) {
+
Preconditions.checkNotNull( plugins, "plugins must not be null" );
Preconditions.checkNotNull( migrationInfoSerialization, "migrationInfoSerialization must not be null" );
this.migrationInfoSerialization = migrationInfoSerialization;
+ this.executionOrder = new ArrayList<>(plugins.size());
+ this.migrationPlugins = new HashMap<>();
+
+
for ( MigrationPlugin plugin : plugins ) {
final String name = plugin.getName();
@@ -68,8 +79,16 @@ public class DataMigrationManagerImpl implements DataMigrationManager {
.getClass().getName() + "' is also trying to implement this name." );
}
- migrationPlugins.put( name, plugin );
+ this.migrationPlugins.put( name, plugin );
+ this.executionOrder.add( plugin );
+
}
+
+ //now sort based on execution order
+
+ Collections.sort(executionOrder, MigrationPluginComparator.INSTANCE);
+
+
}
@@ -79,12 +98,10 @@ public class DataMigrationManagerImpl implements DataMigrationManager {
/**
* Invoke each plugin to attempt a migration
*/
- for(final MigrationPlugin plugin: migrationPlugins.values()){
+ for(final MigrationPlugin plugin: executionOrder){
final ProgressObserver observer = new CassandraProgressObserver(plugin.getName());
plugin.run( observer );
-
-
}
@@ -236,4 +253,26 @@ public class DataMigrationManagerImpl implements DataMigrationManager {
return failed;
}
}
+
+ private final static class MigrationPluginComparator implements Comparator<MigrationPlugin> {
+
+ public static final MigrationPluginComparator INSTANCE = new MigrationPluginComparator();
+
+ @Override
+ public int compare( final MigrationPlugin o1, final MigrationPlugin o2 ) {
+
+ //first one is less
+ if(o1.getPhase().ordinal() < o2.getPhase().ordinal()){
+ return -1;
+ }
+
+ //second one is first
+ if(o2.getPhase().ordinal() < o2.getPhase().ordinal()){
+ return 1;
+ }
+
+ //if our phase for
+ return o1.getName().compareTo( o2.getName() );
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/275e74a5/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/MigrationPlugin.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/MigrationPlugin.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/MigrationPlugin.java
index 897ce27..4953f25 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/MigrationPlugin.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/MigrationPlugin.java
@@ -24,7 +24,6 @@
package org.apache.usergrid.persistence.core.migration.data;
-
/**
* A simple interface to return migration plugins. All versions within this migration plugin should have a name
*/
@@ -48,4 +47,10 @@ public interface MigrationPlugin {
* @return
*/
int getMaxVersion();
+
+ /**
+ * Get the phase of the plugin
+ * @return
+ */
+ PluginPhase getPhase();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/275e74a5/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/PluginPhase.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/PluginPhase.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/PluginPhase.java
new file mode 100644
index 0000000..186ecae
--- /dev/null
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/PluginPhase.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence.core.migration.data;
+
+
+/**
+ * This is the plugin lifeccyle where plugins should be executed. Plugins are executed in the following order
+ *
+ *
+ */
+public enum PluginPhase {
+
+ /**
+ * Runs before any data migration. This is used to prepare our migration subsystem.
+ * For instance, a change in the migration system itself. Most plugins won't need to use this
+ */
+ BOOTSTRAP,
+
+ /**
+ * This is where data migration actually happens. Plugins should be able to run concurrently.
+ * If a plugin has a race condition with another plugin, they should be refactored into a single plugin,
+ * with migration versions encapsulated within the plugin itself
+ */
+ MIGRATE
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/275e74a5/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManagerImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManagerImplTest.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManagerImplTest.java
index 5d516d4..453958e 100644
--- a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManagerImplTest.java
+++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManagerImplTest.java
@@ -21,14 +21,19 @@ package org.apache.usergrid.persistence.core.migration.data;
import java.util.HashSet;
+import java.util.LinkedHashSet;
import java.util.Set;
import org.junit.Test;
+import org.mockito.InOrder;
import org.apache.usergrid.persistence.core.migration.schema.MigrationException;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.inOrder;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@@ -38,11 +43,11 @@ public class DataMigrationManagerImplTest {
@Test
- public void testNoPlugins(){
+ public void testNoPlugins() {
- final Set<MigrationPlugin> plugins = new HashSet<>( );
+ final Set<MigrationPlugin> plugins = new HashSet<>();
- final MigrationInfoSerialization migrationInfoSerialization = mock(MigrationInfoSerialization.class);
+ final MigrationInfoSerialization migrationInfoSerialization = mock( MigrationInfoSerialization.class );
DataMigrationManagerImpl migrationManager = new DataMigrationManagerImpl( plugins, migrationInfoSerialization );
@@ -50,31 +55,30 @@ public class DataMigrationManagerImplTest {
Set<String> pluginNames = migrationManager.getPluginNames();
- assertEquals(0, pluginNames.size());
-
+ assertEquals( 0, pluginNames.size() );
}
-
@Test
public void test2Plugins() throws MigrationException {
- final Set<MigrationPlugin> plugins = new HashSet<>( );
+ final Set<MigrationPlugin> plugins = new HashSet<>();
- MigrationPlugin plugin1 = mock(MigrationPlugin.class);
+ MigrationPlugin plugin1 = mock( MigrationPlugin.class );
+ when( plugin1.getPhase() ).thenReturn( PluginPhase.MIGRATE );
- when(plugin1.getName()).thenReturn("plugin1");
+ when( plugin1.getName() ).thenReturn( "plugin1" );
- MigrationPlugin plugin2 = mock(MigrationPlugin.class);
+ MigrationPlugin plugin2 = mock( MigrationPlugin.class );
+ when( plugin2.getPhase() ).thenReturn( PluginPhase.MIGRATE );
- when(plugin2.getName()).thenReturn("plugin2");
+ when( plugin2.getName() ).thenReturn( "plugin2" );
plugins.add( plugin1 );
plugins.add( plugin2 );
-
- final MigrationInfoSerialization migrationInfoSerialization = mock(MigrationInfoSerialization.class);
+ final MigrationInfoSerialization migrationInfoSerialization = mock( MigrationInfoSerialization.class );
DataMigrationManagerImpl migrationManager = new DataMigrationManagerImpl( plugins, migrationInfoSerialization );
@@ -82,43 +86,39 @@ public class DataMigrationManagerImplTest {
Set<String> pluginNames = migrationManager.getPluginNames();
- assertEquals(2, pluginNames.size());
+ assertEquals( 2, pluginNames.size() );
- assertTrue(pluginNames.contains( "plugin1"));
+ assertTrue( pluginNames.contains( "plugin1" ) );
- assertTrue(pluginNames.contains( "plugin2" ));
+ assertTrue( pluginNames.contains( "plugin2" ) );
//now run them
migrationManager.migrate();
- verify(plugin1).run( any(ProgressObserver.class) );
-
- verify(plugin2).run( any(ProgressObserver.class) );
+ verify( plugin1 ).run( any( ProgressObserver.class ) );
+ verify( plugin2 ).run( any( ProgressObserver.class ) );
}
-
-
@Test
public void testRunning() throws MigrationException {
- final Set<MigrationPlugin> plugins = new HashSet<>( );
+ final Set<MigrationPlugin> plugins = new HashSet<>();
- MigrationPlugin plugin1 = mock(MigrationPlugin.class);
+ MigrationPlugin plugin1 = mock( MigrationPlugin.class );
- when(plugin1.getName()).thenReturn("plugin1");
+ when( plugin1.getName() ).thenReturn( "plugin1" );
+ when( plugin1.getPhase() ).thenReturn( PluginPhase.MIGRATE );
plugins.add( plugin1 );
+ final MigrationInfoSerialization migrationInfoSerialization = mock( MigrationInfoSerialization.class );
- final MigrationInfoSerialization migrationInfoSerialization = mock(MigrationInfoSerialization.class);
-
- when(migrationInfoSerialization.getStatusCode( "plugin1" )).thenReturn( DataMigrationManagerImpl.StatusCode.RUNNING.status );
-
-
+ when( migrationInfoSerialization.getStatusCode( "plugin1" ) )
+ .thenReturn( DataMigrationManagerImpl.StatusCode.RUNNING.status );
DataMigrationManagerImpl migrationManager = new DataMigrationManagerImpl( plugins, migrationInfoSerialization );
@@ -126,23 +126,72 @@ public class DataMigrationManagerImplTest {
boolean status = migrationManager.isRunning();
- assertTrue("Status is set", status);
+ assertTrue( "Status is set", status );
- when(migrationInfoSerialization.getStatusCode( "plugin1" )).thenReturn( DataMigrationManagerImpl.StatusCode.COMPLETE.status );
+ when( migrationInfoSerialization.getStatusCode( "plugin1" ) )
+ .thenReturn( DataMigrationManagerImpl.StatusCode.COMPLETE.status );
status = migrationManager.isRunning();
assertFalse( "Status is not running", status );
- when(migrationInfoSerialization.getStatusCode( "plugin1" )).thenReturn( DataMigrationManagerImpl.StatusCode.ERROR.status );
+ when( migrationInfoSerialization.getStatusCode( "plugin1" ) )
+ .thenReturn( DataMigrationManagerImpl.StatusCode.ERROR.status );
- status = migrationManager.isRunning();
+ status = migrationManager.isRunning();
- assertFalse("Status is not running", status);
+ assertFalse( "Status is not running", status );
}
+ @Test
+ public void testExecutionOrder() throws MigrationException {
+
+
+ //linked hash set is intentional here. For iteration order we can boostrap to come second so we can
+ //verify it was actually run first
+ final Set<MigrationPlugin> plugins = new LinkedHashSet<>();
+
+ MigrationPlugin plugin1 = mock( MigrationPlugin.class );
+ when( plugin1.getPhase() ).thenReturn( PluginPhase.MIGRATE );
+ when( plugin1.getName() ).thenReturn( "plugin1" );
+
+ //boostrap plugin, should run first
+ MigrationPlugin plugin2 = mock( MigrationPlugin.class );
+ when( plugin2.getPhase() ).thenReturn( PluginPhase.BOOTSTRAP );
+
+ when( plugin2.getName() ).thenReturn( "plugin2" );
+
+ plugins.add( plugin1 );
+ plugins.add( plugin2 );
+
+
+ final MigrationInfoSerialization migrationInfoSerialization = mock( MigrationInfoSerialization.class );
+
+
+ DataMigrationManagerImpl migrationManager = new DataMigrationManagerImpl( plugins, migrationInfoSerialization );
+
+
+ Set<String> pluginNames = migrationManager.getPluginNames();
+
+ assertEquals( 2, pluginNames.size() );
+
+ assertTrue( pluginNames.contains( "plugin1" ) );
+
+ assertTrue( pluginNames.contains( "plugin2" ) );
+
+ //now run them
+
+ migrationManager.migrate();
+
+
+
+ //we want to verify the bootsrap plugin was called first
+ InOrder inOrderVerification = inOrder( plugin1, plugin2 );
+ inOrderVerification.verify( plugin2 ).run( any( ProgressObserver.class ) );
+ inOrderVerification.verify( plugin1 ).run( any( ProgressObserver.class ) );
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/275e74a5/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/GraphMigrationPlugin.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/GraphMigrationPlugin.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/GraphMigrationPlugin.java
index 8da102d..22f34ff 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/GraphMigrationPlugin.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/GraphMigrationPlugin.java
@@ -30,10 +30,13 @@ import org.apache.usergrid.persistence.core.migration.data.MigrationInfoSerializ
import org.apache.usergrid.persistence.core.migration.data.AbstractMigrationPlugin;
import org.apache.usergrid.persistence.core.migration.data.DataMigration;
import org.apache.usergrid.persistence.core.migration.data.MigrationDataProvider;
+import org.apache.usergrid.persistence.core.migration.data.PluginPhase;
import com.google.inject.Inject;
import com.google.inject.Singleton;
+import sun.plugin2.main.server.Plugin;
+
/**
* Migration plugin for the collection module
@@ -60,4 +63,8 @@ public class GraphMigrationPlugin extends AbstractMigrationPlugin<GraphNode> {
}
+ @Override
+ public PluginPhase getPhase() {
+ return PluginPhase.MIGRATE;
+ }
}
[10/50] [abbrv] incubator-usergrid git commit: Pushed migration tests
down to core tiers. WIP
Posted by sf...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/802dcdec/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/comparators/TargetDirectedEdgeDescendingComparatorTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/comparators/TargetDirectedEdgeDescendingComparatorTest.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/comparators/TargetDirectedEdgeDescendingComparatorTest.java
index 15df661..4ec1486 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/comparators/TargetDirectedEdgeDescendingComparatorTest.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/comparators/TargetDirectedEdgeDescendingComparatorTest.java
@@ -26,11 +26,12 @@ import java.util.UUID;
import org.junit.Test;
+import org.apache.usergrid.persistence.core.util.IdGenerator;
import org.apache.usergrid.persistence.graph.impl.SimpleMarkedEdge;
import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
-import static org.apache.usergrid.persistence.graph.test.util.EdgeTestUtils.createId;
+import static org.apache.usergrid.persistence.core.util.IdGenerator.createId;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@@ -43,8 +44,8 @@ public class TargetDirectedEdgeDescendingComparatorTest {
@Test
public void sameEdges() {
- final Id sourceId = createId( "source" );
- final Id targetId = createId( "target" );
+ final Id sourceId = IdGenerator.createId( "source" );
+ final Id targetId = IdGenerator.createId( "target" );
final String type = "type";
final long timestamp = 10000;
@@ -65,8 +66,8 @@ public class TargetDirectedEdgeDescendingComparatorTest {
@Test
public void timestampDifferent() {
- final Id sourceId = createId( "source" );
- final Id targetId = createId( "target" );
+ final Id sourceId = IdGenerator.createId( "source" );
+ final Id targetId = IdGenerator.createId( "target" );
final String type = "type";
final long timestamp = 10000;
@@ -88,9 +89,9 @@ public class TargetDirectedEdgeDescendingComparatorTest {
@Test
public void uuidDifferent() {
- final Id sourceId = createId( "source" );
- final Id targetId1 = createId( "target" );
- final Id targetId2 = createId( "target" );
+ final Id sourceId = IdGenerator.createId( "source" );
+ final Id targetId1 = IdGenerator.createId( "target" );
+ final Id targetId2 = IdGenerator.createId( "target" );
final String type = "type";
final long timestamp = 10000;
@@ -114,9 +115,9 @@ public class TargetDirectedEdgeDescendingComparatorTest {
final UUID targetId = UUIDGenerator.newTimeUUID();
- final Id sourceId = createId( "source" );
- final Id targetId1 = createId( targetId, "target1" );
- final Id targetId2 = createId( targetId, "target2" );
+ final Id sourceId = IdGenerator.createId( "source" );
+ final Id targetId1 = IdGenerator.createId( targetId, "target1" );
+ final Id targetId2 = IdGenerator.createId( targetId, "target2" );
final String type = "type";
final long timestamp = 10000;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/802dcdec/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/test/util/EdgeTestUtils.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/test/util/EdgeTestUtils.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/test/util/EdgeTestUtils.java
index ca82b8d..cc0bb51 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/test/util/EdgeTestUtils.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/test/util/EdgeTestUtils.java
@@ -21,8 +21,8 @@ package org.apache.usergrid.persistence.graph.test.util;
import java.util.Random;
-import java.util.UUID;
+import org.apache.usergrid.persistence.core.util.IdGenerator;
import org.apache.usergrid.persistence.graph.Edge;
import org.apache.usergrid.persistence.graph.MarkedEdge;
import org.apache.usergrid.persistence.graph.SearchByEdge;
@@ -36,8 +36,6 @@ import org.apache.usergrid.persistence.graph.impl.SimpleSearchByIdType;
import org.apache.usergrid.persistence.graph.impl.SimpleSearchEdgeType;
import org.apache.usergrid.persistence.graph.impl.SimpleSearchIdType;
import org.apache.usergrid.persistence.model.entity.Id;
-import org.apache.usergrid.persistence.model.entity.SimpleId;
-import org.apache.usergrid.persistence.model.util.UUIDGenerator;
/**
@@ -61,7 +59,7 @@ public class EdgeTestUtils {
* @return an Edge for testing
*/
public static MarkedEdge createEdge( final String sourceType, final String edgeType, final String targetType ) {
- return createEdge( createId( sourceType ), edgeType, createId( targetType ), System.currentTimeMillis() );
+ return createEdge( IdGenerator.createId( sourceType ), edgeType, IdGenerator.createId( targetType ), System.currentTimeMillis() );
}
@@ -76,7 +74,7 @@ public class EdgeTestUtils {
* @return an Edge for testing
*/
public static MarkedEdge createEdge( final String sourceType, final String edgeType, final String targetType, final long timestamp ) {
- return createEdge( createId( sourceType ), edgeType, createId( targetType ), timestamp );
+ return createEdge( IdGenerator.createId( sourceType ), edgeType, IdGenerator.createId( targetType ), timestamp );
}
@@ -92,7 +90,7 @@ public class EdgeTestUtils {
*/
public static MarkedEdge createMarkedEdge( final String sourceType, final String edgeType,
final String targetType ) {
- return createEdge( createId( sourceType ), edgeType, createId( targetType ), System.currentTimeMillis(),
+ return createEdge( IdGenerator.createId( sourceType ), edgeType, IdGenerator.createId( targetType ), System.currentTimeMillis(),
true );
}
@@ -140,25 +138,6 @@ public class EdgeTestUtils {
/**
- * Create the id
- */
- public static Id createId( String type ) {
- return createId(UUIDGenerator.newTimeUUID(), type );
- }
-
-
- /**
- * Generate an ID with the type and id
- *
- * @param id The uuid in the id
- * @param type The type of id
- */
- public static Id createId( UUID id, String type ) {
- return new SimpleId( id, type );
- }
-
-
- /**
*
* @param sourceId
* @param type
[39/50] [abbrv] incubator-usergrid git commit: Removed Hystrix.
Causing thread OOM issues and not really used in production.
Posted by sf...@apache.org.
Removed Hystrix. Causing thread OOM issues and not really used in production.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/5e6ed4a1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/5e6ed4a1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/5e6ed4a1
Branch: refs/heads/USERGRID-480
Commit: 5e6ed4a12c7aa154f1411d85c6fcce56fbcd4367
Parents: b09dc43
Author: Todd Nine <tn...@apigee.com>
Authored: Tue Mar 17 18:23:58 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Tue Mar 17 18:23:58 2015 -0600
----------------------------------------------------------------------
.../core/astyanax/MultiRowColumnIterator.java | 12 ++-
.../core/hystrix/HystrixCassandra.java | 94 --------------------
.../persistence/core/task/TaskExecutor.java | 4 +-
.../graph/impl/GraphManagerImpl.java | 30 +++++--
.../graph/impl/stage/EdgeDeleteRepairImpl.java | 9 +-
.../graph/impl/stage/EdgeMetaRepairImpl.java | 16 +++-
.../impl/stage/NodeDeleteListenerImpl.java | 16 +++-
.../impl/NodeSerializationImpl.java | 36 ++++----
.../shard/count/NodeShardApproximationImpl.java | 4 +-
.../NodeShardCounterSerializationImpl.java | 25 +++---
.../shard/impl/NodeShardAllocationImpl.java | 16 +++-
.../shard/impl/ShardGroupCompactionImpl.java | 64 ++++---------
12 files changed, 128 insertions(+), 198 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5e6ed4a1/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/MultiRowColumnIterator.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/MultiRowColumnIterator.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/MultiRowColumnIterator.java
index fdc4768..667992c 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/MultiRowColumnIterator.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/MultiRowColumnIterator.java
@@ -33,10 +33,8 @@ import java.util.NoSuchElementException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-
-import org.apache.usergrid.persistence.core.hystrix.HystrixCassandra;
-
import com.netflix.astyanax.Keyspace;
+import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
import com.netflix.astyanax.model.Column;
import com.netflix.astyanax.model.ColumnFamily;
import com.netflix.astyanax.model.ColumnList;
@@ -184,7 +182,13 @@ public class MultiRowColumnIterator<R, C, T> implements Iterator<T> {
keyspace.prepareQuery( cf ).setConsistencyLevel( consistencyLevel ).getKeySlice( rowKeys )
.withColumnRange( rangeBuilder.build() );
- final Rows<R, C> result = HystrixCassandra.user( query ).getResult();
+ final Rows<R, C> result;
+ try {
+ result = query.execute().getResult();
+ }
+ catch ( ConnectionException e ) {
+ throw new RuntimeException( "Unable to connect to casandra", e );
+ }
//now aggregate them together
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5e6ed4a1/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/hystrix/HystrixCassandra.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/hystrix/HystrixCassandra.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/hystrix/HystrixCassandra.java
deleted file mode 100644
index ab71782..0000000
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/hystrix/HystrixCassandra.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.persistence.core.hystrix;
-
-
-import com.netflix.astyanax.Execution;
-import com.netflix.astyanax.connectionpool.OperationResult;
-import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
-import com.netflix.hystrix.HystrixCommand;
-import com.netflix.hystrix.HystrixCommandGroupKey;
-import com.netflix.hystrix.HystrixThreadPoolProperties;
-
-
-/**
- * A utility class that creates graph observables wrapped in Hystrix for timeouts and circuit breakers.
- */
-public class HystrixCassandra {
-
-
-
-
- /**
- * Command group used for realtime user commands
- */
- public static final HystrixCommand.Setter
- USER_GROUP = HystrixCommand.Setter.withGroupKey( HystrixCommandGroupKey.Factory.asKey( "user" ) ).andThreadPoolPropertiesDefaults(
- HystrixThreadPoolProperties.Setter().withCoreSize( 1000 ) );
-
- /**
- * Command group for asynchronous operations
- */
- public static final HystrixCommand.Setter
- ASYNC_GROUP = HystrixCommand.Setter.withGroupKey( HystrixCommandGroupKey.Factory.asKey( "async" ) ).andThreadPoolPropertiesDefaults(
- HystrixThreadPoolProperties.Setter().withCoreSize( 1000 ) );
-
-
- /**
- * Execute an user operation
- */
- public static <R> OperationResult<R> user( final Execution<R> execution) {
- return new HystrixCommand<OperationResult<R>>( USER_GROUP ) {
-
- @Override
- protected OperationResult<R> run() {
- try {
- return execution.execute();
- }
- catch ( ConnectionException e ) {
- throw new RuntimeException( e );
- }
- }
- }.execute();
- }
-
-
- /**
- * Execute an an async operation
- */
- public static <R> OperationResult<R> async( final Execution<R> execution) {
-
-
- return new HystrixCommand<OperationResult<R>>( ASYNC_GROUP ) {
-
- @Override
- protected OperationResult<R> run() {
- try {
- return execution.execute();
- }
- catch ( ConnectionException e ) {
- throw new RuntimeException( e );
- }
- }
- }.execute();
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5e6ed4a1/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/task/TaskExecutor.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/task/TaskExecutor.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/task/TaskExecutor.java
index 5e9aa4c..5728d2e 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/task/TaskExecutor.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/task/TaskExecutor.java
@@ -25,7 +25,7 @@ import com.google.common.util.concurrent.ListenableFuture;
/**
* An interface for execution of tasks
*/
-public interface TaskExecutor {
+public interface TaskExecutor {
/**
* Submit the task asynchronously
@@ -37,5 +37,5 @@ public interface TaskExecutor {
* Stop the task executor without waiting for scheduled threads to run
*/
public void shutdown();
-
+
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5e6ed4a1/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java
index 246bbca..26d06ad 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java
@@ -25,14 +25,11 @@ import java.util.List;
import java.util.Map;
import java.util.UUID;
-import com.codahale.metrics.Meter;
-import com.codahale.metrics.Timer;
-import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.usergrid.persistence.core.guice.ProxyImpl;
-import org.apache.usergrid.persistence.core.hystrix.HystrixCassandra;
+import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.apache.usergrid.persistence.core.rx.ObservableIterator;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.util.ValidationUtils;
@@ -54,10 +51,12 @@ import org.apache.usergrid.persistence.graph.serialization.util.GraphValidation;
import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
+import com.codahale.metrics.Meter;
+import com.codahale.metrics.Timer;
import com.google.common.base.Preconditions;
import com.google.inject.Inject;
-import com.google.inject.assistedinject.Assisted;
import com.netflix.astyanax.MutationBatch;
+import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
import rx.Notification;
import rx.Observable;
@@ -202,7 +201,12 @@ public class GraphManagerImpl implements GraphManager {
mutation.mergeShallow( edgeMutation );
- HystrixCassandra.user( mutation );
+ try {
+ mutation.execute();
+ }
+ catch ( ConnectionException e ) {
+ throw new RuntimeException( "Unable to execute mutation", e );
+ }
return edge;
}
@@ -241,7 +245,12 @@ public class GraphManagerImpl implements GraphManager {
LOG.debug("Marking edge {} as deleted to commit log", edge);
- HystrixCassandra.user(edgeMutation);
+ try {
+ edgeMutation.execute();
+ }
+ catch ( ConnectionException e ) {
+ throw new RuntimeException( "Unable to execute mutation", e );
+ }
//HystrixCassandra.async( edgeDeleteListener.receive( scope, markedEdge,
@@ -285,7 +294,12 @@ public class GraphManagerImpl implements GraphManager {
LOG.debug( "Marking node {} as deleted to node mark", node );
- HystrixCassandra.user( nodeMutation );
+ try {
+ nodeMutation.execute();
+ }
+ catch ( ConnectionException e ) {
+ throw new RuntimeException( "Unable to execute mutation", e );
+ }
//HystrixCassandra.async(nodeDeleteListener.receive(scope, id, eventTimestamp )).subscribeOn(
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5e6ed4a1/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeDeleteRepairImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeDeleteRepairImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeDeleteRepairImpl.java
index 0137ba4..7dca0ce 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeDeleteRepairImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeDeleteRepairImpl.java
@@ -26,7 +26,6 @@ import java.util.UUID;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.usergrid.persistence.core.hystrix.HystrixCassandra;
import org.apache.usergrid.persistence.core.rx.ObservableIterator;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.graph.Edge;
@@ -39,6 +38,7 @@ import org.apache.usergrid.persistence.graph.serialization.EdgeSerialization;
import com.google.common.base.Preconditions;
import com.google.inject.Inject;
import com.netflix.astyanax.Keyspace;
+import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
import rx.Observable;
import rx.functions.Action1;
@@ -94,7 +94,12 @@ public class EdgeDeleteRepairImpl implements EdgeDeleteRepair {
//remove from storage
- HystrixCassandra.async(storageSerialization.deleteEdge( scope, edge, timestamp ));
+ try {
+ storageSerialization.deleteEdge( scope, edge, timestamp ).execute();
+ }
+ catch ( ConnectionException e ) {
+ throw new RuntimeException( "Unable to connect to casandra", e );
+ }
}
}
} );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5e6ed4a1/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeMetaRepairImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeMetaRepairImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeMetaRepairImpl.java
index 0e1c4e2..ab141f7 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeMetaRepairImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeMetaRepairImpl.java
@@ -28,7 +28,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.usergrid.persistence.core.guice.ProxyImpl;
-import org.apache.usergrid.persistence.core.hystrix.HystrixCassandra;
import org.apache.usergrid.persistence.core.rx.ObservableIterator;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.util.ValidationUtils;
@@ -47,6 +46,7 @@ import com.google.inject.Inject;
import com.google.inject.Singleton;
import com.netflix.astyanax.Keyspace;
import com.netflix.astyanax.MutationBatch;
+import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
import rx.Observable;
import rx.functions.Action1;
@@ -188,7 +188,12 @@ public class EdgeMetaRepairImpl implements EdgeMetaRepair {
+ "Mutation has {} rows to mutate ",
edgeType, batch.getRowCount() );
- HystrixCassandra.async( batch );
+ try {
+ batch.execute();
+ }
+ catch ( ConnectionException e ) {
+ throw new RuntimeException( "Unable to connect to casandra", e );
+ }
}
}
@@ -219,7 +224,12 @@ public class EdgeMetaRepairImpl implements EdgeMetaRepair {
LOG.debug( "Type {} has no subtypes in use as of maxTimestamp {}. Deleting type.", edgeType,
maxTimestamp );
- HystrixCassandra.async( serialization.removeEdgeType( scope, node, edgeType, maxTimestamp ) );
+ try {
+ serialization.removeEdgeType( scope, node, edgeType, maxTimestamp ).execute();
+ }
+ catch ( ConnectionException e ) {
+ throw new RuntimeException( "Unable to connect to casandra", e );
+ }
}
} );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5e6ed4a1/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/NodeDeleteListenerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/NodeDeleteListenerImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/NodeDeleteListenerImpl.java
index f167f0c..e8c224e 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/NodeDeleteListenerImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/NodeDeleteListenerImpl.java
@@ -29,7 +29,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.usergrid.persistence.core.guice.ProxyImpl;
-import org.apache.usergrid.persistence.core.hystrix.HystrixCassandra;
import org.apache.usergrid.persistence.core.rx.ObservableIterator;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.graph.GraphFig;
@@ -47,6 +46,7 @@ import com.google.common.base.Optional;
import com.google.inject.Inject;
import com.netflix.astyanax.Keyspace;
import com.netflix.astyanax.MutationBatch;
+import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
import rx.Observable;
import rx.functions.Action0;
@@ -129,7 +129,12 @@ public class NodeDeleteListenerImpl implements NodeDeleteListener {
.doOnCompleted( new Action0() {
@Override
public void call() {
- HystrixCassandra.async(nodeSerialization.delete( scope, node, maxVersion.get() ));
+ try {
+ nodeSerialization.delete( scope, node, maxVersion.get()).execute();
+ }
+ catch ( ConnectionException e ) {
+ throw new RuntimeException( "Unable to connect to casandra", e );
+ }
}
} );
}
@@ -210,7 +215,12 @@ public class NodeDeleteListenerImpl implements NodeDeleteListener {
targetNodes.add( new TargetPair( edge.getTargetNode(), edge.getType() ) );
}
- HystrixCassandra.async( batch );
+ try {
+ batch.execute();
+ }
+ catch ( ConnectionException e ) {
+ throw new RuntimeException( "Unable to connect to casandra", e );
+ }
//now delete meta data
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5e6ed4a1/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/NodeSerializationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/NodeSerializationImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/NodeSerializationImpl.java
index 2cc0391..18062c4 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/NodeSerializationImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/NodeSerializationImpl.java
@@ -38,7 +38,6 @@ import org.apache.usergrid.persistence.core.astyanax.MultiTennantColumnFamily;
import org.apache.usergrid.persistence.core.astyanax.MultiTennantColumnFamilyDefinition;
import org.apache.usergrid.persistence.core.astyanax.ScopedRowKeySerializer;
import org.apache.usergrid.persistence.core.astyanax.ScopedRowKey;
-import org.apache.usergrid.persistence.core.hystrix.HystrixCassandra;
import org.apache.usergrid.persistence.core.migration.schema.Migration;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.util.ValidationUtils;
@@ -52,6 +51,7 @@ import com.google.common.base.Preconditions;
import com.google.inject.Singleton;
import com.netflix.astyanax.Keyspace;
import com.netflix.astyanax.MutationBatch;
+import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
import com.netflix.astyanax.connectionpool.exceptions.NotFoundException;
import com.netflix.astyanax.model.Column;
import com.netflix.astyanax.model.Row;
@@ -150,22 +150,23 @@ public class NodeSerializationImpl implements NodeSerialization, Migration {
keyspace.prepareQuery( GRAPH_DELETE ).setConsistencyLevel( fig.getReadCL() );
+
+ Column<Boolean> result = null;
try {
- Column<Boolean> result = HystrixCassandra
- .user( query.getKey( ScopedRowKey.fromKey( scope.getApplication(), node ) ).getColumn( COLUMN_NAME ) )
+ result = query.getKey( ScopedRowKey.fromKey( scope.getApplication(), node ) ).getColumn( COLUMN_NAME ).execute()
.getResult();
-
- return Optional.of( result.getLongValue() );
}
- catch (RuntimeException re ) {
- if(re.getCause().getCause() instanceof NotFoundException) {
- //swallow, there's just no column
- return Optional.absent();
- }
-
- throw re;
+ catch(NotFoundException nfe){
+ //swallow, there's just no column
+ return Optional.absent();
+ }
+ catch ( ConnectionException e ) {
+ throw new RuntimeException( "Unable to connect to casandra", e );
}
+ return Optional.of( result.getLongValue() );
+
+
}
@@ -193,9 +194,14 @@ public class NodeSerializationImpl implements NodeSerialization, Migration {
}
- final Rows<ScopedRowKey<Id>, Boolean> results = HystrixCassandra
- .user( query.getRowSlice( keys ).withColumnSlice( Collections.singletonList( COLUMN_NAME ) ) )
- .getResult();
+ final Rows<ScopedRowKey<Id>, Boolean> results;
+ try {
+ results = query.getRowSlice( keys ).withColumnSlice( Collections.singletonList( COLUMN_NAME )).execute()
+ .getResult();
+ }
+ catch ( ConnectionException e ) {
+ throw new RuntimeException( "Unable to connect to casandra", e );
+ }
for ( Row<ScopedRowKey<Id>, Boolean> row : results ) {
Column<Boolean> column = row.getColumns().getColumnByName( COLUMN_NAME );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5e6ed4a1/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/count/NodeShardApproximationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/count/NodeShardApproximationImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/count/NodeShardApproximationImpl.java
index 47243e5..a47d528 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/count/NodeShardApproximationImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/count/NodeShardApproximationImpl.java
@@ -30,7 +30,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.usergrid.persistence.core.consistency.TimeService;
-import org.apache.usergrid.persistence.core.hystrix.HystrixCassandra;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.graph.GraphFig;
import org.apache.usergrid.persistence.graph.serialization.impl.shard.DirectedEdgeMeta;
@@ -39,6 +38,7 @@ import org.apache.usergrid.persistence.graph.serialization.impl.shard.Shard;
import com.netflix.astyanax.MutationBatch;
import com.netflix.hystrix.HystrixCommand;
+import com.netflix.hystrix.HystrixCommandGroupKey;
import rx.functions.Action0;
import rx.schedulers.Schedulers;
@@ -229,7 +229,7 @@ public class NodeShardApproximationImpl implements NodeShardApproximation {
/**
* Execute the command in hystrix to avoid slamming cassandra
*/
- new HystrixCommand( HystrixCassandra.ASYNC_GROUP ) {
+ new HystrixCommand( HystrixCommandGroupKey.Factory.asKey("BatchCounterRollup") ) {
@Override
protected Void run() throws Exception {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5e6ed4a1/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/count/NodeShardCounterSerializationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/count/NodeShardCounterSerializationImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/count/NodeShardCounterSerializationImpl.java
index 524a0cf..6934275 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/count/NodeShardCounterSerializationImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/count/NodeShardCounterSerializationImpl.java
@@ -33,9 +33,8 @@ import org.apache.usergrid.persistence.core.astyanax.CompositeFieldSerializer;
import org.apache.usergrid.persistence.core.astyanax.IdRowCompositeSerializer;
import org.apache.usergrid.persistence.core.astyanax.MultiTennantColumnFamily;
import org.apache.usergrid.persistence.core.astyanax.MultiTennantColumnFamilyDefinition;
-import org.apache.usergrid.persistence.core.astyanax.ScopedRowKeySerializer;
import org.apache.usergrid.persistence.core.astyanax.ScopedRowKey;
-import org.apache.usergrid.persistence.core.hystrix.HystrixCassandra;
+import org.apache.usergrid.persistence.core.astyanax.ScopedRowKeySerializer;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
import org.apache.usergrid.persistence.graph.GraphFig;
@@ -50,6 +49,7 @@ import com.google.inject.Singleton;
import com.netflix.astyanax.Keyspace;
import com.netflix.astyanax.MutationBatch;
import com.netflix.astyanax.connectionpool.OperationResult;
+import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
import com.netflix.astyanax.connectionpool.exceptions.NotFoundException;
import com.netflix.astyanax.model.Column;
import com.netflix.astyanax.model.CompositeBuilder;
@@ -117,24 +117,19 @@ public class NodeShardCounterSerializationImpl implements NodeShardCounterSerial
final ScopedRowKey rowKey = ScopedRowKey.fromKey( key.scope.getApplication(), key );
+ OperationResult<Column<Boolean>> column = null;
try {
- OperationResult<Column<Boolean>> column = HystrixCassandra.user(
- keyspace.prepareQuery( EDGE_SHARD_COUNTS ).getKey( rowKey ).getColumn( true ) );
-
- return column.getResult().getLongValue();
+ column = keyspace.prepareQuery( EDGE_SHARD_COUNTS ).getKey( rowKey ).getColumn( true ).execute();
}
//column not found, return 0
- catch ( RuntimeException re ) {
-
- final Throwable cause = re.getCause();
-
- if(cause != null && cause.getCause() instanceof NotFoundException) {
- return 0;
- }
-
- throw re;
+ catch ( NotFoundException nfe ) {
+ return 0;
+ }
+ catch ( ConnectionException e ) {
+ throw new RuntimeException( "Unable to read from cassandra", e );
}
+ return column.getResult().getLongValue();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5e6ed4a1/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/NodeShardAllocationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/NodeShardAllocationImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/NodeShardAllocationImpl.java
index 0d34b63..ad64338 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/NodeShardAllocationImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/NodeShardAllocationImpl.java
@@ -27,7 +27,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.usergrid.persistence.core.consistency.TimeService;
-import org.apache.usergrid.persistence.core.hystrix.HystrixCassandra;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.util.ValidationUtils;
import org.apache.usergrid.persistence.graph.GraphFig;
@@ -49,6 +48,7 @@ import com.google.common.base.Optional;
import com.google.common.base.Preconditions;
import com.google.inject.Inject;
import com.netflix.astyanax.MutationBatch;
+import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
import com.netflix.astyanax.util.TimeUUIDUtils;
@@ -110,7 +110,12 @@ public class NodeShardAllocationImpl implements NodeShardAllocation {
final MutationBatch batch = edgeShardSerialization.writeShardMeta( scope, MIN_SHARD, directedEdgeMeta );
- HystrixCassandra.user( batch );
+ try {
+ batch.execute();
+ }
+ catch ( ConnectionException e ) {
+ throw new RuntimeException( "Unable to connect to casandra", e );
+ }
existingShards = Collections.singleton( MIN_SHARD ).iterator();
}
@@ -232,7 +237,12 @@ public class NodeShardAllocationImpl implements NodeShardAllocation {
final MutationBatch batch = this.edgeShardSerialization.writeShardMeta( scope, newShard, directedEdgeMeta );
- HystrixCassandra.user( batch );
+ try {
+ batch.execute();
+ }
+ catch ( ConnectionException e ) {
+ throw new RuntimeException( "Unable to connect to casandra", e );
+ }
return true;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5e6ed4a1/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupCompactionImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupCompactionImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupCompactionImpl.java
index 9f8efc8..6135121 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupCompactionImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupCompactionImpl.java
@@ -29,13 +29,7 @@ import java.util.Iterator;
import java.util.Random;
import java.util.Set;
import java.util.UUID;
-import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.RejectedExecutionHandler;
-import java.util.concurrent.ThreadFactory;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicLong;
import javax.annotation.Nullable;
@@ -43,7 +37,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.usergrid.persistence.core.consistency.TimeService;
-import org.apache.usergrid.persistence.core.hystrix.HystrixCassandra;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.task.Task;
import org.apache.usergrid.persistence.core.task.TaskExecutor;
@@ -74,6 +67,7 @@ import com.google.inject.Inject;
import com.google.inject.Singleton;
import com.netflix.astyanax.Keyspace;
import com.netflix.astyanax.MutationBatch;
+import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
/**
@@ -207,8 +201,8 @@ public class ShardGroupCompactionImpl implements ShardGroupCompaction {
if ( edgeCount % maxWorkSize == 0 ) {
try {
- HystrixCassandra.async( newRowBatch );
- HystrixCassandra.async( deleteRowBatch );
+ newRowBatch.execute();
+ deleteRowBatch.execute();
}
catch ( Throwable t ) {
LOG.error( "Unable to move edges from shard {} to shard {}", sourceShard, targetShard );
@@ -219,8 +213,8 @@ public class ShardGroupCompactionImpl implements ShardGroupCompaction {
try {
- HystrixCassandra.async( newRowBatch );
- HystrixCassandra.async( deleteRowBatch );
+ newRowBatch.execute();
+ deleteRowBatch.execute();
}
catch ( Throwable t ) {
LOG.error( "Unable to move edges to target shard {}", targetShard );
@@ -260,7 +254,12 @@ public class ShardGroupCompactionImpl implements ShardGroupCompaction {
}
- HystrixCassandra.async( shardRemovalRollup );
+ try {
+ shardRemovalRollup.execute();
+ }
+ catch ( ConnectionException e ) {
+ throw new RuntimeException( "Unable to connect to casandra", e );
+ }
LOG.info( "Shard has been fully compacted. Marking shard {} as compacted in Cassandra", targetShard );
@@ -268,7 +267,12 @@ public class ShardGroupCompactionImpl implements ShardGroupCompaction {
//Overwrite our shard index with a newly created one that has been marked as compacted
Shard compactedShard = new Shard( targetShard.getShardIndex(), timeService.getCurrentTime(), true );
final MutationBatch updateMark = edgeShardSerialization.writeShardMeta( scope, compactedShard, edgeMeta );
- HystrixCassandra.async( updateMark );
+ try {
+ updateMark.execute();
+ }
+ catch ( ConnectionException e ) {
+ throw new RuntimeException( "Unable to connect to casandra", e );
+ }
resultBuilder.withCompactedShard( compactedShard );
}
@@ -530,40 +534,6 @@ public class ShardGroupCompactionImpl implements ShardGroupCompaction {
}
- /**
- * Create a thread pool that will reject work if our audit tasks become overwhelmed
- */
- private final class MaxSizeThreadPool extends ThreadPoolExecutor {
-
- public MaxSizeThreadPool( final int workerSize, final int queueLength ) {
- super( 1, workerSize, 30, TimeUnit.SECONDS, new ArrayBlockingQueue<Runnable>( queueLength ),
- new CompactionThreadFactory(), new RejectionLogger() );
- }
- }
-
-
- private final class CompactionThreadFactory implements ThreadFactory {
-
- private final AtomicLong threadCounter = new AtomicLong();
-
-
- @Override
- public Thread newThread( final Runnable r ) {
- final long newValue = threadCounter.incrementAndGet();
-
- return new Thread( r, "Graph-Shard-Compaction-" + newValue );
- }
- }
-
-
- private final class RejectionLogger implements RejectedExecutionHandler {
-
-
- @Override
- public void rejectedExecution( final Runnable r, final ThreadPoolExecutor executor ) {
- LOG.warn( "Audit queue full, rejecting audit task {}", r );
- }
- }
public static final class CompactionResult {
[32/50] [abbrv] incubator-usergrid git commit: Merge branch
'USERGRID-405' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid
into USERGRID-405
Posted by sf...@apache.org.
Merge branch 'USERGRID-405' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into USERGRID-405
Conflicts:
stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/aa01d736
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/aa01d736
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/aa01d736
Branch: refs/heads/USERGRID-480
Commit: aa01d736c2d0e31c73c10d96e7d683468f7c02a0
Parents: ade9714 b635113
Author: Todd Nine <tn...@apigee.com>
Authored: Mon Mar 9 13:36:45 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Mon Mar 9 13:36:45 2015 -0600
----------------------------------------------------------------------
.../Client/IUsergridClient.cs | 2 +-
.../Usergrid.Notifications/Client/PushClient.cs | 24 +++
.../Usergrid.Notifications/MainPage.xaml | 1 +
.../Usergrid.Notifications/MainPage.xaml.cs | 29 ++-
.../main/dist/init_instance/init_rest_server.sh | 3 -
.../src/main/groovy/configure_usergrid.groovy | 6 +-
stack/awscluster/ugcluster-cf.json | 8 +
.../corepersistence/CpEntityManager.java | 109 +++++++++-
.../corepersistence/CpEntityManagerFactory.java | 152 ++++++--------
.../corepersistence/CpRelationManager.java | 39 +++-
.../corepersistence/OrgApplicationCache.java | 67 +++++++
.../OrgApplicationCacheImpl.java | 181 +++++++++++++++++
.../corepersistence/results/EntityVerifier.java | 4 +-
.../results/FilteringLoader.java | 2 +-
.../cassandra/CassandraPersistenceUtils.java | 2 +-
.../main/resources/usergrid-core-context.xml | 14 +-
.../cassandra/EntityManagerFactoryImplIT.java | 1 +
.../impl/migration/GraphMigrationPlugin.java | 1 -
.../persistence/index/IndexBufferProducer.java | 10 +-
.../usergrid/persistence/index/IndexFig.java | 12 +-
.../index/impl/EsEntityIndexBatchImpl.java | 15 +-
.../index/impl/EsEntityIndexFactoryImpl.java | 9 +-
.../index/impl/EsEntityIndexImpl.java | 199 +++++++++++++++----
.../index/impl/EsIndexBufferConsumerImpl.java | 39 +++-
.../index/impl/EsIndexBufferProducerImpl.java | 30 ++-
.../persistence/index/impl/IndexingUtils.java | 7 +-
.../persistence/index/impl/EntityIndexTest.java | 2 +-
.../datagenerators/EntityDataGenerator.scala | 25 +++
.../datagenerators/FeederGenerator.scala | 18 +-
.../usergrid/scenarios/EntityScenarios.scala | 8 +-
.../org/apache/usergrid/settings/Settings.scala | 4 +-
.../simulations/PutCustomEntitySimulation.scala | 75 +++++++
.../org/apache/usergrid/rest/IndexResource.java | 19 ++
.../services/notifications/wns/WNSAdapter.java | 39 ++++
.../resources/usergrid-services-context.xml | 8 +-
35 files changed, 949 insertions(+), 215 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/aa01d736/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------