You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by to...@apache.org on 2015/09/18 20:51:05 UTC
[01/13] usergrid git commit: Initial testing with guice wiring for
test
Repository: usergrid
Updated Branches:
refs/heads/two-dot-o-dev a4fa0941a -> 327ccc647
Initial testing with guice wiring for test
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/b625ed24
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/b625ed24
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/b625ed24
Branch: refs/heads/two-dot-o-dev
Commit: b625ed24a68d9c71fc30694c5dbeda26f1afb4d7
Parents: 2faa805
Author: Todd Nine <tn...@apigee.com>
Authored: Mon Sep 14 17:33:38 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Mon Sep 14 18:14:00 2015 -0600
----------------------------------------------------------------------
.../corepersistence/CpRelationManager.java | 23 ++-
.../asyncevents/AsyncEventService.java | 2 +
.../builder/PipelineBuilderFactory.java | 3 +
.../service/ConnectionScope.java | 53 +++++
.../service/ConnectionService.java | 10 +
.../service/ConnectionServiceImpl.java | 65 +++++-
.../corepersistence/util/CpNamingUtils.java | 13 +-
.../corepersistence/TestCoreModule.java | 34 ++++
.../service/ConnectionServiceImplTest.java | 198 +++++++++++++++++++
stack/core/src/test/resources/log4j.properties | 3 +-
.../impl/TargetIdObservableImpl.java | 13 +-
11 files changed, 397 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/b625ed24/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index 0c3b927..1da6f75 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -63,6 +63,7 @@ import org.apache.usergrid.persistence.graph.GraphManager;
import org.apache.usergrid.persistence.graph.SearchByEdge;
import org.apache.usergrid.persistence.graph.SearchByEdgeType;
import org.apache.usergrid.persistence.graph.impl.SimpleEdge;
+import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdge;
import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdgeType;
import org.apache.usergrid.persistence.graph.impl.SimpleSearchEdgeType;
import org.apache.usergrid.persistence.index.EntityIndex;
@@ -694,9 +695,27 @@ public class CpRelationManager implements RelationManager {
final Edge edge = createConnectionEdge( cpHeadEntity.getId(), connectionType, targetEntity.getId() );
final GraphManager gm = managerCache.getGraphManager( applicationScope );
- gm.writeEdge( edge ).toBlocking().last();
- indexService.queueNewEdge( applicationScope, targetEntity, edge );
+
+ //check if the edge exists
+
+
+ final SearchByEdge searchByEdge = new SimpleSearchByEdge(edge.getSourceNode(), edge.getType(), edge.getTargetNode(), Long.MAX_VALUE, SearchByEdgeType.Order.DESCENDING, Optional.absent() );
+
+
+ //only take 1 and count it. If we don't have anything, create the edge
+ final int count = gm.loadEdgeVersions( searchByEdge ).take( 1 ).count().toBlocking().last();
+
+ if(count == 0) {
+ if(logger.isDebugEnabled()) {
+ logger.debug( "No edge exists between {} and {} of type {}. Creating",
+ new Object[] { edge.getSourceNode(), edge.getTargetNode(), edge.getType() } );
+ }
+
+ gm.writeEdge( edge ).toBlocking().last();
+
+ indexService.queueNewEdge( applicationScope, targetEntity, edge );
+ }
return connection;
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/b625ed24/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventService.java
index 6d51679..ae5688c 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventService.java
@@ -54,6 +54,8 @@ public interface AsyncEventService extends ReIndexAction {
/**
* Fired when a new edge is added to an entity. Such as initial entity creation, adding to a collection, or creating a connection
*
+ * TODO: We shouldn't take an entity here, only the id. It doesn't make sense in a distributed context to pass the entity
+ *
* @param applicationScope
* @param entity
* @param newEdge
http://git-wip-us.apache.org/repos/asf/usergrid/blob/b625ed24/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/PipelineBuilderFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/PipelineBuilderFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/PipelineBuilderFactory.java
index 6cb515b..ada1779 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/PipelineBuilderFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/PipelineBuilderFactory.java
@@ -23,6 +23,9 @@ package org.apache.usergrid.corepersistence.pipeline.builder;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+/**
+ * Factory interface for creating our pipeline builders
+ */
public interface PipelineBuilderFactory {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/b625ed24/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/ConnectionScope.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/ConnectionScope.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/ConnectionScope.java
new file mode 100644
index 0000000..3cc9ae5
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/ConnectionScope.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.corepersistence.service;
+
+
+import java.io.Serializable;
+
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.graph.Edge;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+
+/**
+ * Tuple containing our application scope, and an edge within that applications' graph
+ */
+public class ConnectionScope implements Serializable {
+ private final Edge edge;
+ private final ApplicationScope applicationScope;
+
+
+
+
+ public ConnectionScope( ApplicationScope applicationScope, final Edge edge ) {
+ this.edge = edge;
+ this.applicationScope = applicationScope;
+ }
+
+
+ public Edge getEdge() {
+ return edge;
+ }
+
+
+ public ApplicationScope getApplicationScope() {
+ return applicationScope;
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/b625ed24/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/ConnectionService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/ConnectionService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/ConnectionService.java
index 71a25c9..539c881 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/ConnectionService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/ConnectionService.java
@@ -20,6 +20,7 @@ package org.apache.usergrid.corepersistence.service;
import org.apache.usergrid.corepersistence.pipeline.read.ResultsPage;
import org.apache.usergrid.persistence.ConnectionRef;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
@@ -46,4 +47,13 @@ public interface ConnectionService {
* @return
*/
Observable<ResultsPage<ConnectionRef>> searchConnectionAsRefs( final ConnectionSearch search );
+
+
+ /**
+ * An observable that will remove duplicate edges from the graph that represent connections. All emitted scopes are scopes that have been deleted.
+ *
+ * @param applicationScopeObservable
+ * @return
+ */
+ Observable<ConnectionScope> deDupeConnections(final Observable<ApplicationScope> applicationScopeObservable);
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/b625ed24/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/ConnectionServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/ConnectionServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/ConnectionServiceImpl.java
index c7e0fee..82c1038 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/ConnectionServiceImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/ConnectionServiceImpl.java
@@ -22,7 +22,18 @@ import org.apache.usergrid.corepersistence.pipeline.builder.EntityBuilder;
import org.apache.usergrid.corepersistence.pipeline.builder.IdBuilder;
import org.apache.usergrid.corepersistence.pipeline.builder.PipelineBuilderFactory;
import org.apache.usergrid.corepersistence.pipeline.read.ResultsPage;
+import org.apache.usergrid.corepersistence.rx.impl.AllEntityIdsObservable;
+import org.apache.usergrid.corepersistence.util.CpNamingUtils;
import org.apache.usergrid.persistence.ConnectionRef;
+import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.graph.GraphManager;
+import org.apache.usergrid.persistence.graph.GraphManagerFactory;
+import org.apache.usergrid.persistence.graph.SearchByEdge;
+import org.apache.usergrid.persistence.graph.SearchByEdgeType;
+import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdge;
+import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdgeType;
+import org.apache.usergrid.persistence.graph.impl.SimpleSearchEdgeType;
import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
@@ -37,11 +48,17 @@ import rx.Observable;
public class ConnectionServiceImpl implements ConnectionService {
private final PipelineBuilderFactory pipelineBuilderFactory;
+ private final AllEntityIdsObservable allEntityIdsObservable;
+ private final GraphManagerFactory graphManagerFactory;
@Inject
- public ConnectionServiceImpl( final PipelineBuilderFactory pipelineBuilderFactory ) {
+ public ConnectionServiceImpl( final PipelineBuilderFactory pipelineBuilderFactory,
+ final AllEntityIdsObservable allEntityIdsObservable,
+ final GraphManagerFactory graphManagerFactory ) {
this.pipelineBuilderFactory = pipelineBuilderFactory;
+ this.allEntityIdsObservable = allEntityIdsObservable;
+ this.graphManagerFactory = graphManagerFactory;
}
@@ -117,4 +134,50 @@ public class ConnectionServiceImpl implements ConnectionService {
return results;
}
+
+
+ @Override
+ public Observable<ConnectionScope> deDupeConnections(
+ final Observable<ApplicationScope> applicationScopeObservable ) {
+
+
+ final Observable<EntityIdScope> entityIds =allEntityIdsObservable.getEntities( applicationScopeObservable );
+ //now we have an observable of entityIds. Walk each connection type
+
+ //get all edge types for connections
+ return entityIds.flatMap( entityIdScope -> {
+
+ final ApplicationScope applicationScope = entityIdScope.getApplicationScope();
+ final Id entityId = entityIdScope.getId();
+
+ final GraphManager gm = graphManagerFactory.createEdgeManager(applicationScope );
+
+ return gm.getEdgeTypesFromSource(
+ new SimpleSearchEdgeType( entityId, CpNamingUtils.EDGE_CONN_PREFIX, Optional.absent() ) )
+
+ //now load all edges from this node of this type
+ .flatMap( edgeType -> {
+ final SearchByEdgeType searchByEdge =
+ new SimpleSearchByEdgeType( entityId, edgeType, Long.MAX_VALUE,
+ SearchByEdgeType.Order.ASCENDING, Optional.absent() );
+
+ //load edges from the source the with type specified
+ return gm.loadEdgesFromSource( searchByEdge );
+ } )
+
+ //now that we have a stream of edges, stream all versions
+ .flatMap( edge -> {
+ final SearchByEdge searchByEdge =
+ new SimpleSearchByEdge( edge.getSourceNode(), edge.getType(), edge.getTargetNode(),
+ Long.MAX_VALUE, SearchByEdgeType.Order.ASCENDING, Optional.absent() );
+ return gm.loadEdgeVersions( searchByEdge );
+ } )
+
+ //skip the first version since it's the one we want to retain
+ // validate there is only 1 version of it, delete anything > than the min
+ .skip( 1 )
+ .flatMap( edgeToDelete -> gm.deleteEdge( edgeToDelete ) )
+ .map(deletedEdge -> new ConnectionScope( applicationScope, deletedEdge ) ) ;
+ });
+ }
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/b625ed24/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
index f32e9d7..55bc49a 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
@@ -21,8 +21,6 @@ package org.apache.usergrid.corepersistence.util;
import java.util.UUID;
-import com.google.common.base.Optional;
-import com.google.common.base.Preconditions;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
import org.apache.usergrid.persistence.entities.Application;
@@ -42,7 +40,8 @@ import org.apache.usergrid.persistence.map.impl.MapScopeImpl;
import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.entity.SimpleId;
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
-import org.apache.usergrid.utils.UUIDUtils;
+
+import com.google.common.base.Optional;
/**
@@ -257,6 +256,7 @@ public class CpNamingUtils {
}
+
/**
* Get the application scope from the given uuid
*
@@ -309,6 +309,8 @@ public class CpNamingUtils {
}
+
+
private static boolean isCollectionEdgeType( String type ) {
return type.startsWith( EDGE_COLL_PREFIX );
}
@@ -326,8 +328,5 @@ public class CpNamingUtils {
}
- public static boolean isApplication(Id id) {
- Preconditions.checkNotNull(id);
- return id.getType().equals("application");
- }
+
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/b625ed24/stack/core/src/test/java/org/apache/usergrid/corepersistence/TestCoreModule.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/TestCoreModule.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/TestCoreModule.java
new file mode 100644
index 0000000..32b9845
--- /dev/null
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/TestCoreModule.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.usergrid.corepersistence;
+
+
+import org.apache.usergrid.persistence.core.guice.TestModule;
+
+
+/**
+ * Test guice module for our core guice configuration
+ */
+public class TestCoreModule extends TestModule {
+
+ @Override
+ protected void configure() {
+
+ install( new CoreModule() );
+ }
+}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/b625ed24/stack/core/src/test/java/org/apache/usergrid/corepersistence/service/ConnectionServiceImplTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/service/ConnectionServiceImplTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/service/ConnectionServiceImplTest.java
new file mode 100644
index 0000000..0a29286
--- /dev/null
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/service/ConnectionServiceImplTest.java
@@ -0,0 +1,198 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.usergrid.corepersistence.service;
+
+
+import java.util.List;
+
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+import org.apache.usergrid.corepersistence.TestCoreModule;
+import org.apache.usergrid.corepersistence.util.CpNamingUtils;
+import org.apache.usergrid.persistence.core.guice.MigrationManagerRule;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
+import org.apache.usergrid.persistence.core.test.ITRunner;
+import org.apache.usergrid.persistence.core.test.UseModules;
+import org.apache.usergrid.persistence.graph.Edge;
+import org.apache.usergrid.persistence.graph.GraphManager;
+import org.apache.usergrid.persistence.graph.GraphManagerFactory;
+import org.apache.usergrid.persistence.graph.SearchByEdge;
+import org.apache.usergrid.persistence.graph.SearchByEdgeType;
+import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdge;
+import org.apache.usergrid.persistence.model.entity.Id;
+import org.apache.usergrid.persistence.model.entity.SimpleId;
+
+import com.google.common.base.Optional;
+import com.google.inject.Inject;
+
+import rx.Observable;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+
+@RunWith( ITRunner.class )
+@UseModules( TestCoreModule.class )
+public class ConnectionServiceImplTest {
+
+ @Inject
+ private GraphManagerFactory graphManagerFactory;
+
+ @Inject
+ private ConnectionService connectionService;
+
+ @Inject
+ @Rule
+ public MigrationManagerRule migrationManagerRule;
+
+
+ @Test
+ public void testSingleConnection() {
+
+ final ApplicationScope applicationScope = new ApplicationScopeImpl( new SimpleId( "application" ) );
+
+
+ final GraphManager gm = graphManagerFactory.createEdgeManager( applicationScope );
+
+ //now write a single connection
+
+ final Id source = new SimpleId( "source" );
+
+
+ //add to a collection
+
+ final String collectionName = "testCollection";
+
+ final Edge collectionEdge = CpNamingUtils.createCollectionEdge( applicationScope.getApplication(), collectionName, source );
+
+ final Edge writtenCollection = gm.writeEdge( collectionEdge ).toBlocking().last();
+
+ assertNotNull("Collection edge written", writtenCollection);
+
+
+
+
+ final Id target = new SimpleId( "target" );
+
+ final String connectionType = "testConnection";
+
+ final Edge connectionEdge = CpNamingUtils.createConnectionEdge( source, connectionType, target );
+
+ final Edge writtenConnection = gm.writeEdge( connectionEdge ).toBlocking().last();
+
+
+ //now run the cleanup
+
+ final int count =
+ connectionService.deDupeConnections( Observable.just( applicationScope ) ).count().toBlocking().last();
+
+ assertEquals( "No edges deleted", 0, count );
+
+ //now ensure we can read the edge.
+
+ final SearchByEdge simpleSearchByEdge =
+ new SimpleSearchByEdge( source, connectionEdge.getType(), target, Long.MAX_VALUE,
+ SearchByEdgeType.Order.DESCENDING, Optional.absent() );
+
+ final List<Edge> edges = gm.loadEdgeVersions( simpleSearchByEdge ).toList().toBlocking().last();
+
+ assertEquals( 1, edges.size() );
+
+ assertEquals( writtenConnection, edges.get( 0 ) );
+ }
+
+
+ @Test
+ public void testDuplicateConnections() {
+
+ final ApplicationScope applicationScope = new ApplicationScopeImpl( new SimpleId( "application" ) );
+
+
+ final GraphManager gm = graphManagerFactory.createEdgeManager( applicationScope );
+
+ //now write a single connection
+
+ final Id source = new SimpleId( "source" );
+
+ final Id target = new SimpleId( "target" );
+
+ final String connectionType = "testConnection";
+
+
+ //add to a collection
+
+ final String collectionName = "testCollection";
+
+ final Edge collectionEdge = CpNamingUtils.createCollectionEdge( applicationScope.getApplication(), collectionName, source );
+
+ final Edge writtenCollection = gm.writeEdge( collectionEdge ).toBlocking().last();
+
+ assertNotNull("Collection edge written", writtenCollection);
+
+ //write our first connection
+ final Edge connection1 = CpNamingUtils.createConnectionEdge( source, connectionType, target );
+
+ final Edge written1 = gm.writeEdge( connection1 ).toBlocking().last();
+
+
+ //write the second
+ final Edge connection2 = CpNamingUtils.createConnectionEdge( source, connectionType, target );
+
+ final Edge written2 = gm.writeEdge( connection2 ).toBlocking().last();
+
+
+ //write the 3rd
+ final Edge connection3 = CpNamingUtils.createConnectionEdge( source, connectionType, target );
+
+ final Edge written3 = gm.writeEdge( connection3 ).toBlocking().last();
+
+
+
+
+ //now run the cleanup
+
+ final List<ConnectionScope> deletedConnections =
+ connectionService.deDupeConnections( Observable.just( applicationScope ) ).toList().toBlocking().last();
+
+ assertEquals( "2 edges deleted", 2, deletedConnections.size() );
+
+ //check our oldest was deleted first
+
+ assertEquals(written2, deletedConnections.get( 0 ));
+
+ assertEquals(written3, deletedConnections.get( 1 ));
+
+
+
+ //now ensure we can read the edge.
+
+ final SearchByEdge simpleSearchByEdge =
+ new SimpleSearchByEdge( source, connection1.getType(), target, Long.MAX_VALUE,
+ SearchByEdgeType.Order.DESCENDING, Optional.absent() );
+
+ //check only 1 exists
+ final List<Edge> edges = gm.loadEdgeVersions( simpleSearchByEdge ).toList().toBlocking().last();
+
+ assertEquals( 1, edges.size() );
+
+ assertEquals( written1, edges.get( 0 ) );
+ }
+}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/b625ed24/stack/core/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/stack/core/src/test/resources/log4j.properties b/stack/core/src/test/resources/log4j.properties
index dfcb260..dd29671 100644
--- a/stack/core/src/test/resources/log4j.properties
+++ b/stack/core/src/test/resources/log4j.properties
@@ -18,7 +18,8 @@
# and the pattern to %c instead of %l. (%l is slower.)
# output messages into a rolling log file as well as stdout
-log4j.rootLogger=ERROR,stdout
+#log4j.rootLogger=ERROR,stdout\
+log4j.rootLogger=DEBUG,stdout
# stdout
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
http://git-wip-us.apache.org/repos/asf/usergrid/blob/b625ed24/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/TargetIdObservableImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/TargetIdObservableImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/TargetIdObservableImpl.java
index 82c7d54..bd8cd3c 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/TargetIdObservableImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/TargetIdObservableImpl.java
@@ -55,18 +55,13 @@ public class TargetIdObservableImpl implements TargetIdObservable {
public Observable<Id> getTargetNodes(final GraphManager gm, final Id sourceNode) {
//only search edge types that start with collections
- return edgesFromSourceObservable.edgesFromSourceDescending( gm, sourceNode ).map( new Func1<Edge, Id>() {
+ return edgesFromSourceObservable.edgesFromSourceDescending( gm, sourceNode ).map( edge -> {
+ final Id targetNode = edge.getTargetNode();
+ logger.debug( "Emitting targetId of {}", edge );
- @Override
- public Id call( final Edge edge ) {
- final Id targetNode = edge.getTargetNode();
- logger.debug( "Emitting targetId of {}", edge );
-
-
- return targetNode;
- }
+ return targetNode;
} );
}
}
[02/13] usergrid git commit: Corrects test setup and adds debug
logging
Posted by to...@apache.org.
Corrects test setup and adds debug logging
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/85e1e7c1
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/85e1e7c1
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/85e1e7c1
Branch: refs/heads/two-dot-o-dev
Commit: 85e1e7c1e17470bf012308011950a7ae43565b5f
Parents: b625ed2
Author: Todd Nine <tn...@apigee.com>
Authored: Mon Sep 14 19:07:28 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Mon Sep 14 19:07:28 2015 -0600
----------------------------------------------------------------------
.../service/ConnectionServiceImpl.java | 26 ++++++++++++++++++--
.../service/ConnectionServiceImplTest.java | 12 ++++++---
2 files changed, 33 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/85e1e7c1/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/ConnectionServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/ConnectionServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/ConnectionServiceImpl.java
index 82c1038..41e6f80 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/ConnectionServiceImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/ConnectionServiceImpl.java
@@ -18,6 +18,9 @@
package org.apache.usergrid.corepersistence.service;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import org.apache.usergrid.corepersistence.pipeline.builder.EntityBuilder;
import org.apache.usergrid.corepersistence.pipeline.builder.IdBuilder;
import org.apache.usergrid.corepersistence.pipeline.builder.PipelineBuilderFactory;
@@ -47,6 +50,9 @@ import rx.Observable;
@Singleton
public class ConnectionServiceImpl implements ConnectionService {
+
+ private static final Logger logger = LoggerFactory.getLogger( ConnectionServiceImpl.class );
+
private final PipelineBuilderFactory pipelineBuilderFactory;
private final AllEntityIdsObservable allEntityIdsObservable;
private final GraphManagerFactory graphManagerFactory;
@@ -152,11 +158,16 @@ public class ConnectionServiceImpl implements ConnectionService {
final GraphManager gm = graphManagerFactory.createEdgeManager(applicationScope );
+ logger.debug( "Checking connections of id {} in application {}", entityId, applicationScope );
+
return gm.getEdgeTypesFromSource(
new SimpleSearchEdgeType( entityId, CpNamingUtils.EDGE_CONN_PREFIX, Optional.absent() ) )
//now load all edges from this node of this type
.flatMap( edgeType -> {
+
+ logger.debug( "Found edge of types of {}, searching for edges", edgeType );
+
final SearchByEdgeType searchByEdge =
new SimpleSearchByEdgeType( entityId, edgeType, Long.MAX_VALUE,
SearchByEdgeType.Order.ASCENDING, Optional.absent() );
@@ -167,6 +178,9 @@ public class ConnectionServiceImpl implements ConnectionService {
//now that we have a stream of edges, stream all versions
.flatMap( edge -> {
+
+ logger.debug( "Found edge {}, searching for multiple versions of edge", edge );
+
final SearchByEdge searchByEdge =
new SimpleSearchByEdge( edge.getSourceNode(), edge.getType(), edge.getTargetNode(),
Long.MAX_VALUE, SearchByEdgeType.Order.ASCENDING, Optional.absent() );
@@ -176,8 +190,16 @@ public class ConnectionServiceImpl implements ConnectionService {
//skip the first version since it's the one we want to retain
// validate there is only 1 version of it, delete anything > than the min
.skip( 1 )
- .flatMap( edgeToDelete -> gm.deleteEdge( edgeToDelete ) )
- .map(deletedEdge -> new ConnectionScope( applicationScope, deletedEdge ) ) ;
+ .flatMap( edgeToDelete -> {
+
+ logger.debug( "Deleting edge {}", edgeToDelete );
+
+ //mark the edge and ignore the cleanup result
+ return gm.markEdge( edgeToDelete )
+ //delete the edge
+ .flatMap( edge -> gm.deleteEdge( edgeToDelete ));
+ } )
+ .map( deletedEdge -> new ConnectionScope( applicationScope, deletedEdge ) ) ;
});
}
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/85e1e7c1/stack/core/src/test/java/org/apache/usergrid/corepersistence/service/ConnectionServiceImplTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/service/ConnectionServiceImplTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/service/ConnectionServiceImplTest.java
index 0a29286..e03a9b3 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/service/ConnectionServiceImplTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/service/ConnectionServiceImplTest.java
@@ -132,9 +132,6 @@ public class ConnectionServiceImplTest {
final Id source = new SimpleId( "source" );
- final Id target = new SimpleId( "target" );
-
- final String connectionType = "testConnection";
//add to a collection
@@ -147,6 +144,15 @@ public class ConnectionServiceImplTest {
assertNotNull("Collection edge written", writtenCollection);
+
+ //now write 3 connections between the same nodes.
+
+ final Id target = new SimpleId( "target" );
+
+ final String connectionType = "testConnection";
+
+
+
//write our first connection
final Edge connection1 = CpNamingUtils.createConnectionEdge( source, connectionType, target );
[05/13] usergrid git commit: Reduces logging and adds filter into
delete listener
Posted by to...@apache.org.
Reduces logging and adds filter into delete listener
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/b7a22c2d
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/b7a22c2d
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/b7a22c2d
Branch: refs/heads/two-dot-o-dev
Commit: b7a22c2d51602e32692b22ad584ce128b1592ed0
Parents: 6233bf0
Author: Todd Nine <tn...@apigee.com>
Authored: Tue Sep 15 17:27:14 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Tue Sep 15 17:27:14 2015 -0600
----------------------------------------------------------------------
stack/core/src/test/resources/log4j.properties | 4 ++--
.../persistence/graph/impl/stage/EdgeDeleteRepairImpl.java | 2 +-
2 files changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/b7a22c2d/stack/core/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/stack/core/src/test/resources/log4j.properties b/stack/core/src/test/resources/log4j.properties
index 93951fe..4267293 100644
--- a/stack/core/src/test/resources/log4j.properties
+++ b/stack/core/src/test/resources/log4j.properties
@@ -19,7 +19,7 @@
# output messages into a rolling log file as well as stdout
#log4j.rootLogger=ERROR,stdout\
-log4j.rootLogger=DEBUG,stdout
+log4j.rootLogger=INFO,stdout
# stdout
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
@@ -48,7 +48,7 @@ log4j.logger.org.apache.usergrid.corepersistence.migration=WARN
#Debug our queries
#log4j.logger.org.apache.usergrid.persistence.index.impl.EsApplicationEntityIndexImpl=DEBUG
-log4j.logger.org.apache.usergrid.corepersistence.service.ConnectionServiceImpl=DEBUG
+#log4j.logger.org.apache.usergrid.corepersistence.service.ConnectionServiceImpl=DEBUG
#log4j.logger.org.apache.usergrid.persistence.index.impl=DEBUG
#log4j.logger.org.apache.usergrid.corepersistence.CpSetup=INFO
http://git-wip-us.apache.org/repos/asf/usergrid/blob/b7a22c2d/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeDeleteRepairImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeDeleteRepairImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeDeleteRepairImpl.java
index 643dcc1..a87a079 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeDeleteRepairImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeDeleteRepairImpl.java
@@ -79,7 +79,7 @@ public class EdgeDeleteRepairImpl implements EdgeDeleteRepair {
//merge source and target then deal with the distinct values
- return Observable.just( edge )
+ return Observable.just( edge ).filter( markedEdge-> markedEdge.isDeleted() )
.doOnNext( markedEdge -> {
//it's still in the same state as it was when we queued it. Remove it
LOG.info( "Removing edge {} ", markedEdge );
[10/13] usergrid git commit: Fixes incorrect name issue
Posted by to...@apache.org.
Fixes incorrect name issue
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/901d7c6f
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/901d7c6f
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/901d7c6f
Branch: refs/heads/two-dot-o-dev
Commit: 901d7c6f76fa8143c3d19d5810ca925c0cf1d5b7
Parents: 2689a01
Author: Todd Nine <tn...@apigee.com>
Authored: Thu Sep 17 16:41:08 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Thu Sep 17 16:41:08 2015 -0600
----------------------------------------------------------------------
.../usergrid/corepersistence/migration/CoreMigrationPlugin.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/901d7c6f/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/CoreMigrationPlugin.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/CoreMigrationPlugin.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/CoreMigrationPlugin.java
index b69451a..b7529f8 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/CoreMigrationPlugin.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/CoreMigrationPlugin.java
@@ -36,7 +36,7 @@ import com.google.inject.Singleton;
@Singleton
public class CoreMigrationPlugin extends AbstractMigrationPlugin {
- public static final String PLUGIN_NAME = "collections-entity-data";
+ public static final String PLUGIN_NAME = "core-data";
[13/13] usergrid git commit: Merge branch '2.1-release' of
https://git-wip-us.apache.org/repos/asf/usergrid into two-dot-o-dev
Posted by to...@apache.org.
Merge branch '2.1-release' of https://git-wip-us.apache.org/repos/asf/usergrid into two-dot-o-dev
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/327ccc64
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/327ccc64
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/327ccc64
Branch: refs/heads/two-dot-o-dev
Commit: 327ccc6479b9cb6d85b52536ded7ec7f2dff2780
Parents: a4fa094 0c12abb
Author: Todd Nine <tn...@apigee.com>
Authored: Fri Sep 18 12:50:57 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Fri Sep 18 12:50:57 2015 -0600
----------------------------------------------------------------------
.../usergrid/corepersistence/CoreModule.java | 51 +++-
.../corepersistence/CpRelationManager.java | 23 +-
.../asyncevents/AsyncEventService.java | 2 +
.../migration/CoreMigration.java | 45 +---
.../migration/CoreMigrationPlugin.java | 38 ++-
.../migration/DeDupConnectionDataMigration.java | 93 ++++++++
.../migration/EntityTypeMappingMigration.java | 104 --------
.../migration/MigrationModuleVersionPlugin.java | 5 -
.../corepersistence/migration/Versions.java | 31 ---
.../builder/PipelineBuilderFactory.java | 3 +
.../service/ConnectionScope.java | 53 +++++
.../service/ConnectionService.java | 10 +
.../service/ConnectionServiceImpl.java | 89 ++++++-
.../corepersistence/util/CpNamingUtils.java | 13 +-
.../corepersistence/TestCoreModule.java | 34 +++
.../DeDupConnectionDataMigrationTest.java | 145 ++++++++++++
.../migration/EntityTypeMappingMigrationIT.java | 125 ----------
.../service/ConnectionServiceImplTest.java | 235 +++++++++++++++++++
stack/core/src/test/resources/log4j.properties | 6 +-
.../serialization/impl/SerializationModule.java | 4 +-
.../migration/CollectionMigrationPlugin.java | 8 +-
.../migration/MvccEntityDataMigrationImpl.java | 10 +-
...ctMvccEntityDataMigrationV1ToV3ImplTest.java | 39 +--
.../migration/data/AbstractMigrationPlugin.java | 17 +-
.../core/migration/data/DataMigration.java | 11 +-
.../migration/data/MigrationDataProvider.java | 2 +-
.../core/migration/data/MigrationPlugin.java | 4 +-
.../migration/schema/MigrationManagerFig.java | 1 +
.../migration/data/TestProgressObserver.java | 10 +-
.../persistence/graph/guice/GraphModule.java | 4 +-
.../graph/impl/GraphManagerImpl.java | 4 +-
.../impl/stage/EdgeDeleteListenerImpl.java | 4 +-
.../graph/impl/stage/EdgeDeleteRepairImpl.java | 61 ++---
.../impl/TargetIdObservableImpl.java | 13 +-
.../impl/migration/EdgeDataMigrationImpl.java | 12 +-
.../impl/migration/GraphMigrationPlugin.java | 8 +-
.../impl/shard/impl/EdgeSearcher.java | 14 +-
.../impl/ShardedEdgeSerializationImpl.java | 59 +++--
.../shard/impl/serialize/EdgeSerializer.java | 21 +-
.../persistence/graph/GraphManagerIT.java | 73 +++++-
.../migration/EdgeDataMigrationImplTest.java | 59 +++--
.../persistence/index/guice/IndexModule.java | 4 +-
.../index/migration/EsIndexMigrationPlugin.java | 8 +-
43 files changed, 1038 insertions(+), 517 deletions(-)
----------------------------------------------------------------------
[08/13] usergrid git commit: Fixes test
Posted by to...@apache.org.
Fixes test
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/acbf3a17
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/acbf3a17
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/acbf3a17
Branch: refs/heads/two-dot-o-dev
Commit: acbf3a17216180f67f0f6c9f390e740efd55b223
Parents: d971ec5
Author: Todd Nine <tn...@apigee.com>
Authored: Thu Sep 17 11:46:45 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Thu Sep 17 11:46:45 2015 -0600
----------------------------------------------------------------------
...ctMvccEntityDataMigrationV1ToV3ImplTest.java | 34 ++++++++++++++------
1 file changed, 25 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/acbf3a17/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/AbstractMvccEntityDataMigrationV1ToV3ImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/AbstractMvccEntityDataMigrationV1ToV3ImplTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/AbstractMvccEntityDataMigrationV1ToV3ImplTest.java
index f51b339..9be979b 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/AbstractMvccEntityDataMigrationV1ToV3ImplTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/AbstractMvccEntityDataMigrationV1ToV3ImplTest.java
@@ -30,7 +30,10 @@ import org.apache.usergrid.persistence.collection.MvccEntity;
import org.apache.usergrid.persistence.collection.guice.TestCollectionModule;
import org.apache.usergrid.persistence.collection.mvcc.entity.impl.MvccEntityImpl;
import org.apache.usergrid.persistence.collection.serialization.MvccEntitySerializationStrategy;
+import org.apache.usergrid.persistence.collection.serialization.MvccLogEntrySerializationStrategy;
+import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
import org.apache.usergrid.persistence.collection.serialization.impl.CollectionDataVersions;
+import org.apache.usergrid.persistence.collection.serialization.impl.MvccEntitySerializationStrategyV3Impl;
import org.apache.usergrid.persistence.core.guice.DataMigrationResetRule;
import org.apache.usergrid.persistence.core.guice.MigrationManagerRule;
import org.apache.usergrid.persistence.core.migration.data.DataMigrationManager;
@@ -48,6 +51,7 @@ import org.apache.usergrid.persistence.model.entity.SimpleId;
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
import com.google.inject.Inject;
+import com.netflix.astyanax.Keyspace;
import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
import net.jcip.annotations.NotThreadSafe;
@@ -63,7 +67,8 @@ import static org.junit.Assert.assertTrue;
@NotThreadSafe
@RunWith( ITRunner.class )
@UseModules( { TestCollectionModule.class } )
-public abstract class AbstractMvccEntityDataMigrationV1ToV3ImplTest implements DataMigrationResetRule.DataMigrationManagerProvider {
+public abstract class AbstractMvccEntityDataMigrationV1ToV3ImplTest
+ implements DataMigrationResetRule.DataMigrationManagerProvider {
@Inject
@@ -76,11 +81,19 @@ public abstract class AbstractMvccEntityDataMigrationV1ToV3ImplTest implements D
@Inject
- public MvccEntityDataMigrationImpl mvccEntityDataMigrationImpl;
+ public VersionedMigrationSet<MvccEntitySerializationStrategy> versions;
@Inject
- public VersionedMigrationSet<MvccEntitySerializationStrategy> versions;
+ public Keyspace keyspace;
+ @Inject
+ public VersionedMigrationSet<MvccEntitySerializationStrategy> allVersions;
+ @Inject
+ public MvccEntitySerializationStrategyV3Impl mvccEntitySerializationStrategyV3;
+ @Inject
+ public UniqueValueSerializationStrategy uniqueValueSerializationStrategy;
+ @Inject
+ public MvccLogEntrySerializationStrategy mvccLogEntrySerializationStrategy;
/**
* Rule to do the resets we need
@@ -121,20 +134,26 @@ public abstract class AbstractMvccEntityDataMigrationV1ToV3ImplTest implements D
Observable.just( new EntityIdScope( scope, entity1.getId() ), new EntityIdScope( scope, entity2.getId() ) );
+ final MigrationDataProvider<EntityIdScope> migrationProvider = () -> entityIdScope;
+
final TestProgressObserver progressObserver = new TestProgressObserver();
final CollectionDataVersions startVersion = getSourceVersion();
final MigrationRelationship<MvccEntitySerializationStrategy> tuple =
- versions.getMigrationRelationship( startVersion.getVersion() );
+ versions.getMigrationRelationship( startVersion.getVersion() );
assertEquals( "Same instance for from", v1Impl.getClass(), tuple.from.getClass() );
assertEquals( "Same instance for to", v3Impl.getClass(), tuple.to.getClass() );
+
+ MvccEntityDataMigrationImpl mvccEntityDataMigrationImpl = new MvccEntityDataMigrationImpl(keyspace, allVersions, mvccEntitySerializationStrategyV3, uniqueValueSerializationStrategy, mvccLogEntrySerializationStrategy, migrationProvider);
+
+
//now migration
- final int newVersion = mvccEntityDataMigrationImpl
- .migrate( startVersion.getVersion(), progressObserver );
+ final int newVersion =
+ mvccEntityDataMigrationImpl.migrate( startVersion.getVersion(), progressObserver );
final CollectionDataVersions expectedVersion = expectedTargetVersion();
@@ -183,19 +202,16 @@ public abstract class AbstractMvccEntityDataMigrationV1ToV3ImplTest implements D
/**
* Get the expected source mvcc implementation for this test
- * @return
*/
protected abstract MvccEntitySerializationStrategy getExpectedSourceImpl();
/**
* Get the expected target mvcc for this test
- * @return
*/
protected abstract MvccEntitySerializationStrategy getExpectedTargetImpl();
/**
* Get the expected start version
- * @return
*/
protected abstract CollectionDataVersions getSourceVersion();
[12/13] usergrid git commit: Merge branch 'USERGRID-1018' into
2.1-release
Posted by to...@apache.org.
Merge branch 'USERGRID-1018' into 2.1-release
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/0c12abb1
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/0c12abb1
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/0c12abb1
Branch: refs/heads/two-dot-o-dev
Commit: 0c12abb154c85ea5ccfd936490a2f84b0ffe5d24
Parents: 61dd559 0d4dd11
Author: Todd Nine <tn...@apigee.com>
Authored: Fri Sep 18 10:48:34 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Fri Sep 18 10:48:34 2015 -0600
----------------------------------------------------------------------
.../usergrid/corepersistence/CoreModule.java | 51 +++-
.../corepersistence/CpRelationManager.java | 23 +-
.../asyncevents/AsyncEventService.java | 2 +
.../migration/CoreMigration.java | 45 +---
.../migration/CoreMigrationPlugin.java | 38 ++-
.../migration/DeDupConnectionDataMigration.java | 93 ++++++++
.../migration/EntityTypeMappingMigration.java | 104 --------
.../migration/MigrationModuleVersionPlugin.java | 5 -
.../corepersistence/migration/Versions.java | 31 ---
.../builder/PipelineBuilderFactory.java | 3 +
.../service/ConnectionScope.java | 53 +++++
.../service/ConnectionService.java | 10 +
.../service/ConnectionServiceImpl.java | 89 ++++++-
.../corepersistence/util/CpNamingUtils.java | 13 +-
.../corepersistence/TestCoreModule.java | 34 +++
.../DeDupConnectionDataMigrationTest.java | 145 ++++++++++++
.../migration/EntityTypeMappingMigrationIT.java | 125 ----------
.../service/ConnectionServiceImplTest.java | 235 +++++++++++++++++++
stack/core/src/test/resources/log4j.properties | 6 +-
.../serialization/impl/SerializationModule.java | 4 +-
.../migration/CollectionMigrationPlugin.java | 8 +-
.../migration/MvccEntityDataMigrationImpl.java | 10 +-
...ctMvccEntityDataMigrationV1ToV3ImplTest.java | 39 +--
.../migration/data/AbstractMigrationPlugin.java | 17 +-
.../core/migration/data/DataMigration.java | 11 +-
.../migration/data/MigrationDataProvider.java | 2 +-
.../core/migration/data/MigrationPlugin.java | 4 +-
.../migration/schema/MigrationManagerFig.java | 1 +
.../migration/data/TestProgressObserver.java | 10 +-
.../persistence/graph/guice/GraphModule.java | 4 +-
.../graph/impl/GraphManagerImpl.java | 4 +-
.../impl/stage/EdgeDeleteListenerImpl.java | 4 +-
.../graph/impl/stage/EdgeDeleteRepairImpl.java | 61 ++---
.../impl/TargetIdObservableImpl.java | 13 +-
.../impl/migration/EdgeDataMigrationImpl.java | 12 +-
.../impl/migration/GraphMigrationPlugin.java | 8 +-
.../impl/shard/impl/EdgeSearcher.java | 14 +-
.../impl/ShardedEdgeSerializationImpl.java | 59 +++--
.../shard/impl/serialize/EdgeSerializer.java | 21 +-
.../persistence/graph/GraphManagerIT.java | 73 +++++-
.../migration/EdgeDataMigrationImplTest.java | 59 +++--
.../persistence/index/guice/IndexModule.java | 4 +-
.../index/migration/EsIndexMigrationPlugin.java | 8 +-
43 files changed, 1038 insertions(+), 517 deletions(-)
----------------------------------------------------------------------
[04/13] usergrid git commit: Fixes incorrect Cassandra range scanning
in serialization with min/max values
Posted by to...@apache.org.
Fixes incorrect Cassandra range scanning in serialization with min/max values
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/6233bf0b
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/6233bf0b
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/6233bf0b
Branch: refs/heads/two-dot-o-dev
Commit: 6233bf0b23e6f86b0d57607d2dd423599c27e726
Parents: 13d2594
Author: Todd Nine <tn...@apigee.com>
Authored: Tue Sep 15 17:20:24 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Tue Sep 15 17:20:24 2015 -0600
----------------------------------------------------------------------
.../service/ConnectionServiceImpl.java | 30 +++++-----
.../service/ConnectionServiceImplTest.java | 19 ++++---
.../core/astyanax/MultiRowColumnIterator.java | 2 +-
.../graph/impl/GraphManagerImpl.java | 4 +-
.../impl/shard/impl/EdgeSearcher.java | 14 ++++-
.../impl/ShardedEdgeSerializationImpl.java | 59 ++++++++++++++------
.../shard/impl/serialize/EdgeSerializer.java | 21 ++++++-
.../persistence/graph/GraphManagerIT.java | 4 +-
8 files changed, 105 insertions(+), 48 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/6233bf0b/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/ConnectionServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/ConnectionServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/ConnectionServiceImpl.java
index d60426c..1b36321 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/ConnectionServiceImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/ConnectionServiceImpl.java
@@ -170,7 +170,7 @@ public class ConnectionServiceImpl implements ConnectionService {
final SearchByEdgeType searchByEdge =
new SimpleSearchByEdgeType( entityId, edgeType, Long.MAX_VALUE,
- SearchByEdgeType.Order.ASCENDING, Optional.absent() );
+ SearchByEdgeType.Order.DESCENDING, Optional.absent() );
//load edges from the source the with type specified
return gm.loadEdgesFromSource( searchByEdge );
@@ -182,21 +182,25 @@ public class ConnectionServiceImpl implements ConnectionService {
logger.debug( "Found edge {}, searching for multiple versions of edge", edge );
final SearchByEdge searchByEdge =
- new SimpleSearchByEdge( edge.getSourceNode(), edge.getType(), edge.getTargetNode(),
- Long.MAX_VALUE, SearchByEdgeType.Order.ASCENDING, Optional.absent() );
- return gm.loadEdgeVersions( searchByEdge );
- } )
+ new SimpleSearchByEdge( edge.getSourceNode(), edge.getType(), edge.getTargetNode(), 0,
+ SearchByEdgeType.Order.ASCENDING, Optional.absent() );
+ return gm.loadEdgeVersions( searchByEdge )
+ //skip the first version since it's the one we want to retain
+ .skip( 1 )
+ //mark for deletion
+ .flatMap( edgeToDelete -> {
- //skip the first version since it's the one we want to retain
- // validate there is only 1 version of it, delete anything > than the min
- .skip( 1 ).flatMap( edgeToDelete -> {
+ logger.debug( "Deleting edge {}", edgeToDelete );
- logger.debug( "Deleting edge {}", edgeToDelete );
+ //mark the edge and ignore the cleanup result
+ return gm.markEdge( edgeToDelete );
+ } )
+ //mark all versions, then on the last, delete them from cass
+ .flatMap( lastMarkedEdge -> gm.deleteEdge( lastMarkedEdge ) );
+
+
+ // validate there is only 1 version of it, delete anything > than the min
- //mark the edge and ignore the cleanup result
- return gm.markEdge( edgeToDelete )
- //delete the edge
- .flatMap( edge -> gm.deleteEdge( edgeToDelete ) );
} ).map( deletedEdge -> new ConnectionScope( applicationScope, deletedEdge ) );
} );
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/6233bf0b/stack/core/src/test/java/org/apache/usergrid/corepersistence/service/ConnectionServiceImplTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/service/ConnectionServiceImplTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/service/ConnectionServiceImplTest.java
index 152721a..a2c18b1 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/service/ConnectionServiceImplTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/service/ConnectionServiceImplTest.java
@@ -49,6 +49,7 @@ import rx.Observable;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
@RunWith( ITRunner.class )
@@ -179,19 +180,20 @@ public class ConnectionServiceImplTest {
final Edge written3 = gm.writeEdge( connection3 ).toBlocking().last();
+
logger.info( "Wrote edge 3 with edge {}", written3 );
+
+ assertTrue( "Expected edge timestamp to be in order", written1.getTimestamp() <= written2.getTimestamp() );
+ assertTrue( "Expected edge timestamp to be in order", written2.getTimestamp() <= written3.getTimestamp() );
+
//now run the cleanup
final List<ConnectionScope> deletedConnections =
connectionService.deDupeConnections( Observable.just( applicationScope ) ).toList().toBlocking().last();
-// assertEquals( "2 edges deleted", 2, deletedConnections.size() );
-
//check our oldest was deleted first
-
-
assertEdgeData( written2, deletedConnections.get( 0 ).getEdge() );
assertEdgeData( written3, deletedConnections.get( 1 ).getEdge() );
@@ -221,12 +223,13 @@ public class ConnectionServiceImplTest {
* @param asserted
*/
private void assertEdgeData(final Edge expected, final Edge asserted){
- assertEquals("SourceId the same", expected.getSourceNode(), expected.getTargetNode());
- assertEquals("TargetId the same", expected.getTargetNode(), expected.getTargetNode());
+ assertEquals("SourceId the same", expected.getSourceNode(), asserted.getSourceNode());
+
+ assertEquals("TargetId the same", expected.getTargetNode(), asserted.getTargetNode());
- assertEquals("Type the same", expected.getType(), expected.getType());
+ assertEquals("Type the same", expected.getType(), asserted.getType());
- assertEquals("Timestamp the same", expected.getTimestamp(), expected.getTimestamp());
+ assertEquals("Timestamp the same", expected.getTimestamp(), asserted.getTimestamp());
}
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/6233bf0b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/MultiRowColumnIterator.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/MultiRowColumnIterator.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/MultiRowColumnIterator.java
index 667992c..16249ae 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/MultiRowColumnIterator.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/MultiRowColumnIterator.java
@@ -201,7 +201,7 @@ public class MultiRowColumnIterator<R, C, T> implements Iterator<T> {
final List<T> mergedResults;
if ( containsSingleRowOnly( result ) ) {
- mergedResults = singleRowResult( result );
+ mergedResults = singleRowResult( result );
}
else {
mergedResults = mergeResults( result, selectSize );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/6233bf0b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java
index 8c64fa1..e119c59 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java
@@ -217,7 +217,7 @@ public class GraphManagerImpl implements GraphManager {
protected Iterator<MarkedEdge> getIterator() {
return storageEdgeSerialization.getEdgeVersions( scope,
new SimpleSearchByEdge( edge.getSourceNode(), edge.getType(), edge.getTargetNode(),
- Long.MAX_VALUE, SearchByEdgeType.Order.ASCENDING, Optional.absent() ) );
+ Long.MAX_VALUE, SearchByEdgeType.Order.DESCENDING, Optional.absent() ) );
}
} ).filter( markedEdge -> markedEdge.isDeleted() ).flatMap( marked ->
//fire our delete listener and wait for the results
@@ -458,7 +458,7 @@ public class GraphManagerImpl implements GraphManager {
final long edgeTimestamp = edge.getTimestamp();
//our edge needs to not be deleted and have a version that's > max Version
- if ( edge.isDeleted() || Long.compare( edgeTimestamp, maxVersion ) > 0 ) {
+ if ( edge.isDeleted() ) {
return false;
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/6233bf0b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/EdgeSearcher.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/EdgeSearcher.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/EdgeSearcher.java
index 8bcefbb..4d02ba9 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/EdgeSearcher.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/EdgeSearcher.java
@@ -1,6 +1,7 @@
package org.apache.usergrid.persistence.graph.serialization.impl.shard.impl;
+import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Comparator;
@@ -13,6 +14,7 @@ import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.graph.Edge;
import org.apache.usergrid.persistence.graph.MarkedEdge;
import org.apache.usergrid.persistence.graph.SearchByEdgeType;
+import org.apache.usergrid.persistence.graph.serialization.impl.shard.DirectedEdge;
import org.apache.usergrid.persistence.graph.serialization.impl.shard.Shard;
import com.google.common.base.Optional;
@@ -107,12 +109,12 @@ public abstract class EdgeSearcher<R, C, T> implements ColumnParser<C, T>, Colum
if ( last.isPresent() ) {
C sourceEdge = createColumn( last.get() );
-
rangeBuilder.setStart( sourceEdge, getSerializer() );
+ }else {
+ setTimeScan( rangeBuilder );
}
-
- setRangeOptions(rangeBuilder);
+ setRangeOptions( rangeBuilder );
}
@@ -122,6 +124,7 @@ public abstract class EdgeSearcher<R, C, T> implements ColumnParser<C, T>, Colum
final boolean reversed = order == SearchByEdgeType.Order.ASCENDING;
rangeBuilder.setReversed( reversed );
+
}
@@ -153,6 +156,11 @@ public abstract class EdgeSearcher<R, C, T> implements ColumnParser<C, T>, Colum
*/
protected abstract C createColumn( final T last );
+ /**
+ * Set the time scan into the range builder
+ * @param rangeBuilder
+ */
+ protected abstract void setTimeScan(final RangeBuilder rangeBuilder);
/**
* Create an edge to return to the user based on the directed edge provided
http://git-wip-us.apache.org/repos/asf/usergrid/blob/6233bf0b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardedEdgeSerializationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardedEdgeSerializationImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardedEdgeSerializationImpl.java
index 13f7427..2ef50f5 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardedEdgeSerializationImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardedEdgeSerializationImpl.java
@@ -21,8 +21,8 @@
package org.apache.usergrid.persistence.graph.serialization.impl.shard.impl;
+import java.nio.ByteBuffer;
import java.util.Collection;
-import java.util.Comparator;
import java.util.Iterator;
import java.util.UUID;
@@ -54,14 +54,14 @@ import org.apache.usergrid.persistence.graph.serialization.impl.shard.ShardedEdg
import org.apache.usergrid.persistence.graph.serialization.impl.shard.impl.comparators.DescendingTimestampComparator;
import org.apache.usergrid.persistence.graph.serialization.impl.shard.impl.comparators.OrderedComparator;
import org.apache.usergrid.persistence.graph.serialization.impl.shard.impl.comparators
- .SourceDirectedEdgeDescendingComparator;
+ .SourceDirectedEdgeDescendingComparator;
import org.apache.usergrid.persistence.graph.serialization.impl.shard.impl.comparators
- .TargetDirectedEdgeDescendingComparator;
+ .TargetDirectedEdgeDescendingComparator;
+import org.apache.usergrid.persistence.graph.serialization.impl.shard.impl.serialize.EdgeSerializer;
import org.apache.usergrid.persistence.graph.serialization.util.GraphValidation;
import org.apache.usergrid.persistence.model.entity.Id;
import com.google.common.base.Function;
-import com.google.common.base.Optional;
import com.google.inject.Singleton;
import com.netflix.astyanax.Keyspace;
import com.netflix.astyanax.MutationBatch;
@@ -388,19 +388,6 @@ public class ShardedEdgeSerializationImpl implements ShardedEdgeSerialization {
}
- @Override
- public void buildRange( final RangeBuilder builder ) {
-
-
- if ( last.isPresent() ) {
- super.buildRange( builder );
- return;
- }
-
- //start seeking at a value < our max version
- builder.setStart( maxTimestamp );
- }
-
@Override
protected EdgeRowKey generateRowKey( long shard ) {
@@ -415,6 +402,13 @@ public class ShardedEdgeSerializationImpl implements ShardedEdgeSerialization {
@Override
+ protected void setTimeScan( final RangeBuilder rangeBuilder ) {
+ //start seeking at a value < our max version
+ rangeBuilder.setStart( maxTimestamp );
+ }
+
+
+ @Override
protected MarkedEdge createEdge( final Long column, final boolean marked ) {
return new SimpleMarkedEdge( sourceId, type, targetId, column.longValue(), marked );
}
@@ -472,6 +466,14 @@ public class ShardedEdgeSerializationImpl implements ShardedEdgeSerialization {
@Override
+ protected void setTimeScan( final RangeBuilder rangeBuilder ) {
+ final ByteBuffer buffer = EdgeSerializer.INSTANCE.fromTimeRange( maxTimestamp );
+
+ rangeBuilder.setStart( buffer );
+ }
+
+
+ @Override
protected MarkedEdge createEdge( final DirectedEdge edge, final boolean marked ) {
return new SimpleMarkedEdge( sourceId, type, edge.id, edge.timestamp, marked );
}
@@ -526,6 +528,14 @@ public class ShardedEdgeSerializationImpl implements ShardedEdgeSerialization {
@Override
+ protected void setTimeScan( final RangeBuilder rangeBuilder ) {
+ final ByteBuffer buffer = EdgeSerializer.INSTANCE.fromTimeRange( maxTimestamp );
+
+ rangeBuilder.setStart( buffer );
+ }
+
+
+ @Override
protected MarkedEdge createEdge( final DirectedEdge edge, final boolean marked ) {
return new SimpleMarkedEdge( targetId, type, edge.id, edge.timestamp, marked );
}
@@ -574,6 +584,14 @@ public class ShardedEdgeSerializationImpl implements ShardedEdgeSerialization {
@Override
+ protected void setTimeScan( final RangeBuilder rangeBuilder ) {
+ final ByteBuffer buffer = EdgeSerializer.INSTANCE.fromTimeRange( maxTimestamp );
+
+ rangeBuilder.setStart( buffer );
+ }
+
+
+ @Override
protected MarkedEdge createEdge( final DirectedEdge edge, final boolean marked ) {
return new SimpleMarkedEdge( edge.id, type, targetId, edge.timestamp, marked );
}
@@ -627,6 +645,13 @@ public class ShardedEdgeSerializationImpl implements ShardedEdgeSerialization {
@Override
+ protected void setTimeScan( final RangeBuilder rangeBuilder ) {
+ final ByteBuffer buffer = EdgeSerializer.INSTANCE.fromTimeRange( maxTimestamp );
+
+ rangeBuilder.setStart( buffer );
+ }
+
+ @Override
protected MarkedEdge createEdge( final DirectedEdge edge, final boolean marked ) {
return new SimpleMarkedEdge( edge.id, type, targetId, edge.timestamp, marked );
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/6233bf0b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/serialize/EdgeSerializer.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/serialize/EdgeSerializer.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/serialize/EdgeSerializer.java
index 590cf35..e652c73 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/serialize/EdgeSerializer.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/serialize/EdgeSerializer.java
@@ -35,14 +35,16 @@ import com.netflix.astyanax.serializers.LongSerializer;
/**
- * Serializes to a source->target edge Note that we cannot set the edge type on de-serialization. Only the target
- * Id and version.
+ * Serializes to a source->target edge Note that we cannot set the edge type on de-serialization. Only the target Id
+ * and version.
*/
public class EdgeSerializer extends AbstractSerializer<DirectedEdge> {
private static final IdColDynamicCompositeSerializer ID_COL_SERIALIZER = IdColDynamicCompositeSerializer.get();
private static final LongSerializer LONG_SERIALIZER = LongSerializer.get();
+ public static final EdgeSerializer INSTANCE = new EdgeSerializer();
+
@Override
public ByteBuffer toByteBuffer( final DirectedEdge edge ) {
@@ -74,4 +76,19 @@ public class EdgeSerializer extends AbstractSerializer<DirectedEdge> {
return new DirectedEdge( id, timestamp );
}
+
+
+ /**
+ * Create a scan range that represents the timestamp of the edge
+ * @param timestamp
+ * @return
+ */
+ public ByteBuffer fromTimeRange( final long timestamp ) {
+ DynamicComposite composite = new DynamicComposite();
+
+ composite.addComponent( timestamp, LONG_SERIALIZER );
+
+
+ return composite.serialize();
+ }
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/6233bf0b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerIT.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerIT.java
index 4b2ffc8..eaded15 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerIT.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerIT.java
@@ -2353,7 +2353,7 @@ public class GraphManagerIT {
final SearchByEdge searchDescending =
- new SimpleSearchByEdge( edge1.getSourceNode(), edge1.getType(), edge1.getTargetNode(),Long.MAX_VALUE,
+ new SimpleSearchByEdge( edge1.getSourceNode(), edge1.getType(), edge1.getTargetNode(), Long.MAX_VALUE,
SearchByEdgeType.Order.DESCENDING, Optional.<Edge>absent() );
final Observable<Edge> edgesDescending = gm.loadEdgeVersions( searchDescending );
@@ -2373,7 +2373,7 @@ public class GraphManagerIT {
//now search ascending
final SearchByEdge searchAscending =
- new SimpleSearchByEdge( edge1.getSourceNode(), edge1.getType(), edge1.getTargetNode(),Long.MAX_VALUE,
+ new SimpleSearchByEdge( edge1.getSourceNode(), edge1.getType(), edge1.getTargetNode(), 0,
SearchByEdgeType.Order.ASCENDING, Optional.<Edge>absent() );
Observable<Edge> edgesAscending = gm.loadEdgeVersions( searchAscending );
[11/13] usergrid git commit: Fixes issues found during review
Posted by to...@apache.org.
Fixes issues found during review
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/0d4dd116
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/0d4dd116
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/0d4dd116
Branch: refs/heads/two-dot-o-dev
Commit: 0d4dd1161e0ffacbfe056de205da0860652b0e17
Parents: 901d7c6
Author: Todd Nine <tn...@apigee.com>
Authored: Thu Sep 17 17:10:24 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Thu Sep 17 17:10:24 2015 -0600
----------------------------------------------------------------------
.../usergrid/corepersistence/CoreModule.java | 6 ------
.../migration/DeDupConnectionDataMigration.java | 22 ++++++++++----------
.../DeDupConnectionDataMigrationTest.java | 7 ++++---
.../core/astyanax/MultiRowColumnIterator.java | 2 +-
.../persistence/graph/GraphManagerIT.java | 6 +++---
5 files changed, 19 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/0d4dd116/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
index 11707be..486aa6f 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
@@ -190,12 +190,6 @@ public class CoreModule extends AbstractModule {
bind( StatusService.class ).to( StatusServiceImpl.class );
- /**
- * Install migration services
- */
-
- //migrations
- //we want to make sure our generics are retained, so we use a typeliteral
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/0d4dd116/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/DeDupConnectionDataMigration.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/DeDupConnectionDataMigration.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/DeDupConnectionDataMigration.java
index f65864a..bec8d6c 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/DeDupConnectionDataMigration.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/DeDupConnectionDataMigration.java
@@ -56,24 +56,23 @@ public class DeDupConnectionDataMigration implements DataMigration {
observer.start();
- connectionService.deDupeConnections( allApplicationsObservable.getData() ).reduce( 0l, (count, deDuped) ->{
+ connectionService.deDupeConnections( allApplicationsObservable.getData() ).reduce( 0l, ( count, deDuped ) -> {
- final long newCount = count+1;
+ final long newCount = count + 1;
/**
* Update our progress observer
*/
- if(newCount % UPDATE_COUNT == 0){
+ if ( newCount % UPDATE_COUNT == 0 ) {
logger.info( "De duped {} edges", newCount );
- observer.update( migrationVersion, String.format("De duped %d edges", newCount) );
+ observer.update( migrationVersion, String.format( "De duped %d edges", newCount ) );
}
return newCount;
-
- }).doOnNext( total -> {
- logger.info("Completed de-duping {} edges", total );
- observer.complete();;
- }).subscribe();
+ } ).doOnNext( total -> {
+ logger.info( "Completed de-duping {} edges", total );
+ observer.complete();
+ } ).subscribe();
return migrationVersion;
@@ -82,12 +81,13 @@ public class DeDupConnectionDataMigration implements DataMigration {
@Override
public boolean supports( final int currentVersion ) {
- return currentVersion == getMaxVersion() - 1;
+ return currentVersion <= getMaxVersion() - 1;
}
@Override
public int getMaxVersion() {
- return 1;
+ //needs to be 2 b/c our obsolete EntityTypeMappingMigration was 1
+ return 2;
}
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/0d4dd116/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/DeDupConnectionDataMigrationTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/DeDupConnectionDataMigrationTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/DeDupConnectionDataMigrationTest.java
index c84ad2b..aad6259 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/DeDupConnectionDataMigrationTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/DeDupConnectionDataMigrationTest.java
@@ -49,8 +49,6 @@ public class DeDupConnectionDataMigrationTest {
public void testVersion() {
//mock up an initial system state
- final int version = 0;
-
final ConnectionService connectionService = mock( ConnectionService.class );
final AllApplicationsObservable allApplicationsObservable = mock( AllApplicationsObservable.class );
@@ -60,7 +58,10 @@ public class DeDupConnectionDataMigrationTest {
new DeDupConnectionDataMigration( connectionService, allApplicationsObservable );
- assertTrue( plugin.supports( version ) );
+ //anything less than 2 should be supported
+ assertTrue( plugin.supports( 0 ) );
+
+ assertTrue( plugin.supports( 1 ) );
assertFalse( plugin.supports( plugin.getMaxVersion() ) );
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/0d4dd116/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/MultiRowColumnIterator.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/MultiRowColumnIterator.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/MultiRowColumnIterator.java
index 16249ae..667992c 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/MultiRowColumnIterator.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/MultiRowColumnIterator.java
@@ -201,7 +201,7 @@ public class MultiRowColumnIterator<R, C, T> implements Iterator<T> {
final List<T> mergedResults;
if ( containsSingleRowOnly( result ) ) {
- mergedResults = singleRowResult( result );
+ mergedResults = singleRowResult( result );
}
else {
mergedResults = mergeResults( result, selectSize );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/0d4dd116/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerIT.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerIT.java
index eaded15..eda3a02 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerIT.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerIT.java
@@ -2339,15 +2339,15 @@ public class GraphManagerIT {
//write 3 edges with 3 different timestamp
final Edge edge1 = createEdge( sourceId, "test", target, 1, false );
- gm.writeEdge( edge1 ).toBlocking().last();
+ gm.writeEdge( edge1 ).subscribe();
final Edge edge2 = createEdge( sourceId, "test", target, 2, false );
- gm.writeEdge( edge2 ).toBlocking().last();
+ gm.writeEdge( edge2 ).subscribe();
final Edge edge3 = createEdge( sourceId, "test", target, 3, false );
- gm.writeEdge( edge3 ).toBlocking().last();
+ gm.writeEdge( edge3 ).subscribe();
//now test retrieving it
[03/13] usergrid git commit: Added test to prove issue in graph. Need
to fix version load.
Posted by to...@apache.org.
Added test to prove issue in graph. Need to fix version load.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/13d25941
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/13d25941
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/13d25941
Branch: refs/heads/two-dot-o-dev
Commit: 13d2594196bd6315af4305e95e8314cbb600ee3e
Parents: 85e1e7c
Author: Todd Nine <tn...@apigee.com>
Authored: Tue Sep 15 13:34:34 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Tue Sep 15 13:35:21 2015 -0600
----------------------------------------------------------------------
.../service/ConnectionServiceImpl.java | 26 ++++---
.../service/ConnectionServiceImplTest.java | 36 ++++++++--
stack/core/src/test/resources/log4j.properties | 3 +-
.../migration/schema/MigrationManagerFig.java | 1 +
.../impl/stage/EdgeDeleteListenerImpl.java | 4 +-
.../graph/impl/stage/EdgeDeleteRepairImpl.java | 61 +++++-----------
.../persistence/graph/GraphManagerIT.java | 73 +++++++++++++++++++-
7 files changed, 136 insertions(+), 68 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/13d25941/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/ConnectionServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/ConnectionServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/ConnectionServiceImpl.java
index 41e6f80..d60426c 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/ConnectionServiceImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/ConnectionServiceImpl.java
@@ -147,18 +147,18 @@ public class ConnectionServiceImpl implements ConnectionService {
final Observable<ApplicationScope> applicationScopeObservable ) {
- final Observable<EntityIdScope> entityIds =allEntityIdsObservable.getEntities( applicationScopeObservable );
+ final Observable<EntityIdScope> entityIds = allEntityIdsObservable.getEntities( applicationScopeObservable );
//now we have an observable of entityIds. Walk each connection type
//get all edge types for connections
- return entityIds.flatMap( entityIdScope -> {
+ return entityIds.flatMap( entityIdScope -> {
final ApplicationScope applicationScope = entityIdScope.getApplicationScope();
final Id entityId = entityIdScope.getId();
- final GraphManager gm = graphManagerFactory.createEdgeManager(applicationScope );
+ final GraphManager gm = graphManagerFactory.createEdgeManager( applicationScope );
- logger.debug( "Checking connections of id {} in application {}", entityId, applicationScope );
+ logger.debug( "Checking connections of id {} in application {}", entityId, applicationScope );
return gm.getEdgeTypesFromSource(
new SimpleSearchEdgeType( entityId, CpNamingUtils.EDGE_CONN_PREFIX, Optional.absent() ) )
@@ -176,7 +176,7 @@ public class ConnectionServiceImpl implements ConnectionService {
return gm.loadEdgesFromSource( searchByEdge );
} )
- //now that we have a stream of edges, stream all versions
+ //now that we have a stream of edges, stream all versions
.flatMap( edge -> {
logger.debug( "Found edge {}, searching for multiple versions of edge", edge );
@@ -187,19 +187,17 @@ public class ConnectionServiceImpl implements ConnectionService {
return gm.loadEdgeVersions( searchByEdge );
} )
- //skip the first version since it's the one we want to retain
- // validate there is only 1 version of it, delete anything > than the min
- .skip( 1 )
- .flatMap( edgeToDelete -> {
+ //skip the first version since it's the one we want to retain
+ // validate there is only 1 version of it, delete anything > than the min
+ .skip( 1 ).flatMap( edgeToDelete -> {
logger.debug( "Deleting edge {}", edgeToDelete );
//mark the edge and ignore the cleanup result
return gm.markEdge( edgeToDelete )
- //delete the edge
- .flatMap( edge -> gm.deleteEdge( edgeToDelete ));
- } )
- .map( deletedEdge -> new ConnectionScope( applicationScope, deletedEdge ) ) ;
- });
+ //delete the edge
+ .flatMap( edge -> gm.deleteEdge( edgeToDelete ) );
+ } ).map( deletedEdge -> new ConnectionScope( applicationScope, deletedEdge ) );
+ } );
}
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/13d25941/stack/core/src/test/java/org/apache/usergrid/corepersistence/service/ConnectionServiceImplTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/service/ConnectionServiceImplTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/service/ConnectionServiceImplTest.java
index e03a9b3..152721a 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/service/ConnectionServiceImplTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/service/ConnectionServiceImplTest.java
@@ -23,6 +23,8 @@ import java.util.List;
import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.RunWith;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.apache.usergrid.corepersistence.TestCoreModule;
import org.apache.usergrid.corepersistence.util.CpNamingUtils;
@@ -53,6 +55,9 @@ import static org.junit.Assert.assertNotNull;
@UseModules( TestCoreModule.class )
public class ConnectionServiceImplTest {
+
+ private static final Logger logger = LoggerFactory.getLogger(ConnectionServiceImplTest.class);
+
@Inject
private GraphManagerFactory graphManagerFactory;
@@ -158,19 +163,23 @@ public class ConnectionServiceImplTest {
final Edge written1 = gm.writeEdge( connection1 ).toBlocking().last();
+ logger.info( "Wrote edge 1 with edge {}", written1 );
+
//write the second
final Edge connection2 = CpNamingUtils.createConnectionEdge( source, connectionType, target );
final Edge written2 = gm.writeEdge( connection2 ).toBlocking().last();
+ logger.info( "Wrote edge 2 with edge {}", written2 );
+
//write the 3rd
final Edge connection3 = CpNamingUtils.createConnectionEdge( source, connectionType, target );
final Edge written3 = gm.writeEdge( connection3 ).toBlocking().last();
-
+ logger.info( "Wrote edge 3 with edge {}", written3 );
//now run the cleanup
@@ -178,13 +187,16 @@ public class ConnectionServiceImplTest {
final List<ConnectionScope> deletedConnections =
connectionService.deDupeConnections( Observable.just( applicationScope ) ).toList().toBlocking().last();
- assertEquals( "2 edges deleted", 2, deletedConnections.size() );
+// assertEquals( "2 edges deleted", 2, deletedConnections.size() );
//check our oldest was deleted first
- assertEquals(written2, deletedConnections.get( 0 ));
- assertEquals(written3, deletedConnections.get( 1 ));
+ assertEdgeData( written2, deletedConnections.get( 0 ).getEdge() );
+
+ assertEdgeData( written3, deletedConnections.get( 1 ).getEdge() );
+
+ assertEquals( "2 edges deleted", 2, deletedConnections.size() );
@@ -201,4 +213,20 @@ public class ConnectionServiceImplTest {
assertEquals( written1, edges.get( 0 ) );
}
+
+
+ /**
+ * Compares edges based on their sourceId, type, targetId and timestamp. It ignores the deleted flag
+ * @param expected
+ * @param asserted
+ */
+ private void assertEdgeData(final Edge expected, final Edge asserted){
+ assertEquals("SourceId the same", expected.getSourceNode(), expected.getTargetNode());
+ assertEquals("TargetId the same", expected.getTargetNode(), expected.getTargetNode());
+
+ assertEquals("Type the same", expected.getType(), expected.getType());
+
+ assertEquals("Timestamp the same", expected.getTimestamp(), expected.getTimestamp());
+
+ }
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/13d25941/stack/core/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/stack/core/src/test/resources/log4j.properties b/stack/core/src/test/resources/log4j.properties
index dd29671..93951fe 100644
--- a/stack/core/src/test/resources/log4j.properties
+++ b/stack/core/src/test/resources/log4j.properties
@@ -47,7 +47,8 @@ log4j.logger.org.apache.usergrid.persistence.PerformanceEntityRebuildIndexTest=D
log4j.logger.org.apache.usergrid.corepersistence.migration=WARN
#Debug our queries
-log4j.logger.org.apache.usergrid.persistence.index.impl.EsApplicationEntityIndexImpl=DEBUG
+#log4j.logger.org.apache.usergrid.persistence.index.impl.EsApplicationEntityIndexImpl=DEBUG
+log4j.logger.org.apache.usergrid.corepersistence.service.ConnectionServiceImpl=DEBUG
#log4j.logger.org.apache.usergrid.persistence.index.impl=DEBUG
#log4j.logger.org.apache.usergrid.corepersistence.CpSetup=INFO
http://git-wip-us.apache.org/repos/asf/usergrid/blob/13d25941/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/schema/MigrationManagerFig.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/schema/MigrationManagerFig.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/schema/MigrationManagerFig.java
index b95338a..d8f3d1f 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/schema/MigrationManagerFig.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/schema/MigrationManagerFig.java
@@ -35,6 +35,7 @@ public interface MigrationManagerFig extends GuicyFig {
String getStrategyClass();
@Key( "collections.keyspace.strategy.options" )
+ @Default("replication_factor:1")
String getStrategyOptions();
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/13d25941/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeDeleteListenerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeDeleteListenerImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeDeleteListenerImpl.java
index 22a5ad0..fa289bd 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeDeleteListenerImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeDeleteListenerImpl.java
@@ -30,6 +30,7 @@ import com.google.inject.Singleton;
import rx.Observable;
import rx.functions.Func1;
import rx.functions.Func2;
+import rx.observables.MathObservable;
/**
@@ -69,8 +70,7 @@ public class EdgeDeleteListenerImpl implements EdgeDeleteListener {
Observable<Integer> targetDelete = edgeMetaRepair
.repairTargets( scope, edge.getTargetNode(), edge.getType(), maxTimestamp );
- return Observable.zip( sourceDelete, targetDelete,
- ( sourceCount, targetCount ) -> sourceCount + targetCount );
+ return MathObservable.sumInteger( Observable.merge( sourceDelete, targetDelete ) );
} );
}
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/13d25941/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeDeleteRepairImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeDeleteRepairImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeDeleteRepairImpl.java
index 9b84978..643dcc1 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeDeleteRepairImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeDeleteRepairImpl.java
@@ -79,52 +79,25 @@ public class EdgeDeleteRepairImpl implements EdgeDeleteRepair {
//merge source and target then deal with the distinct values
- return Observable.just( edge ).flatMap( new Func1<MarkedEdge, Observable<? extends MarkedEdge>>() {
- @Override
- public Observable<? extends MarkedEdge> call( final MarkedEdge edge ) {
-
- return getEdgeVersions( scope, edge, storageSerialization ).take( 1 )
- .doOnNext( new Action1<MarkedEdge>() {
- @Override
- public void call( final MarkedEdge markedEdge ) {
- //it's still in the same state as it was when we queued it. Remove it
- if ( edge.equals( markedEdge ) ) {
- LOG.info( "Removing edge {} ", edge );
-
- //remove from the commit log
-
-
- //remove from storage
- try {
- storageSerialization.deleteEdge( scope, edge, timestamp ).execute();
- }
- catch ( ConnectionException e ) {
- throw new RuntimeException( "Unable to connect to casandra", e );
- }
- }
- }
- } );
- }
- } );
+ return Observable.just( edge )
+ .doOnNext( markedEdge -> {
+ //it's still in the same state as it was when we queued it. Remove it
+ LOG.info( "Removing edge {} ", markedEdge );
+
+ //remove from the commit log
+
+
+ //remove from storage
+ try {
+ storageSerialization.deleteEdge( scope, markedEdge, timestamp ).execute();
+ }
+ catch ( ConnectionException e ) {
+ throw new RuntimeException( "Unable to connect to casandra", e );
+ }
+ }
+ );
}
- /**
- * Get all edge versions <= the specified max from the source
- */
- private Observable<MarkedEdge> getEdgeVersions( final ApplicationScope scope, final Edge edge,
- final EdgeSerialization serialization ) {
- return Observable.create( new ObservableIterator<MarkedEdge>( "edgeVersions" ) {
- @Override
- protected Iterator<MarkedEdge> getIterator() {
-
- final SimpleSearchByEdge search =
- new SimpleSearchByEdge( edge.getSourceNode(), edge.getType(), edge.getTargetNode(),
- edge.getTimestamp(), SearchByEdgeType.Order.DESCENDING, Optional.<Edge>absent() );
-
- return serialization.getEdgeVersions( scope, search );
- }
- } );
- }
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/13d25941/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerIT.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerIT.java
index d416b1d..4b2ffc8 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerIT.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerIT.java
@@ -20,9 +20,9 @@ package org.apache.usergrid.persistence.graph;
import java.util.Iterator;
+import java.util.List;
import java.util.concurrent.TimeoutException;
-import com.google.common.base.Optional;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
@@ -35,11 +35,14 @@ import org.apache.usergrid.persistence.core.test.ITRunner;
import org.apache.usergrid.persistence.core.test.UseModules;
import org.apache.usergrid.persistence.core.util.IdGenerator;
import org.apache.usergrid.persistence.graph.guice.TestGraphModule;
+import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdge;
+import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdgeType;
import org.apache.usergrid.persistence.graph.impl.SimpleSearchEdgeType;
import org.apache.usergrid.persistence.graph.impl.SimpleSearchIdType;
import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.entity.SimpleId;
+import com.google.common.base.Optional;
import com.google.inject.Inject;
import rx.Observable;
@@ -836,7 +839,7 @@ public class GraphManagerIT {
//now load the next page
//tests that even if a prefix is specified, the last takes precedence
- edgeTypes = new SimpleSearchEdgeType( testTargetEdge.getSourceNode(), null, Optional.fromNullable("test") );
+ edgeTypes = new SimpleSearchEdgeType( testTargetEdge.getSourceNode(), null, Optional.fromNullable( "test" ) );
edges = gm.getEdgeTypesFromSource( edgeTypes );
@@ -914,7 +917,7 @@ public class GraphManagerIT {
//now load the next page
- edgeTypes = new SimpleSearchEdgeType( testTargetEdge2.getTargetNode(), null, Optional.fromNullable("test") );
+ edgeTypes = new SimpleSearchEdgeType( testTargetEdge2.getTargetNode(), null, Optional.fromNullable( "test" ) );
edges = gm.getEdgeTypesToTarget( edgeTypes );
@@ -2321,6 +2324,70 @@ public class GraphManagerIT {
em.markEdge( null );
}
+
+
+ @Test
+ public void testReadMultipleVersionOrder() {
+
+ GraphManager gm = emf.createEdgeManager( scope );
+
+ final Id sourceId = createId( "source" );
+
+ final Id target = createId( "target" );
+
+
+ //write 3 edges with 3 different timestamp
+ final Edge edge1 = createEdge( sourceId, "test", target, 1, false );
+
+ gm.writeEdge( edge1 ).toBlocking().last();
+
+ final Edge edge2 = createEdge( sourceId, "test", target, 2, false );
+
+ gm.writeEdge( edge2 ).toBlocking().last();
+
+ final Edge edge3 = createEdge( sourceId, "test", target, 3, false );
+
+ gm.writeEdge( edge3 ).toBlocking().last();
+
+ //now test retrieving it
+
+
+ final SearchByEdge searchDescending =
+ new SimpleSearchByEdge( edge1.getSourceNode(), edge1.getType(), edge1.getTargetNode(),Long.MAX_VALUE,
+ SearchByEdgeType.Order.DESCENDING, Optional.<Edge>absent() );
+
+ final Observable<Edge> edgesDescending = gm.loadEdgeVersions( searchDescending );
+
+ //search descending
+ final List<Edge> descending = edgesDescending.toList().toBlocking().single();
+
+ assertEquals( "Correct size returned", 3, descending.size() );
+
+ assertEquals( "Correct edges returned", edge3, descending.get( 0 ) );
+
+ assertEquals( "Correct edges returned", edge2, descending.get( 1 ) );
+
+ assertEquals( "Correct edges returned", edge1, descending.get( 2 ) );
+
+
+ //now search ascending
+
+ final SearchByEdge searchAscending =
+ new SimpleSearchByEdge( edge1.getSourceNode(), edge1.getType(), edge1.getTargetNode(),Long.MAX_VALUE,
+ SearchByEdgeType.Order.ASCENDING, Optional.<Edge>absent() );
+
+ Observable<Edge> edgesAscending = gm.loadEdgeVersions( searchAscending );
+
+ List<Edge> ascending = edgesAscending.toList().toBlocking().single();
+
+ assertEquals( "Correct size returned", 3, ascending.size() );
+
+ assertEquals( "Correct edges returned", edge1, ascending.get( 0 ) );
+
+ assertEquals( "Correct edges returned", edge2, ascending.get( 1 ) );
+
+ assertEquals( "Correct edges returned", edge3, ascending.get( 2 ) );
+ }
}
[09/13] usergrid git commit: Removes obsolete plugins.
Posted by to...@apache.org.
Removes obsolete plugins.
Adds tests and implements entity-data plugin.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/2689a015
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/2689a015
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/2689a015
Branch: refs/heads/two-dot-o-dev
Commit: 2689a015938e6b05ceb6f6cc2c228f61cb5e3b17
Parents: acbf3a1
Author: Todd Nine <tn...@apigee.com>
Authored: Thu Sep 17 16:17:43 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Thu Sep 17 16:17:43 2015 -0600
----------------------------------------------------------------------
.../usergrid/corepersistence/CoreModule.java | 53 +++++--
.../migration/CoreMigration.java | 45 ++----
.../migration/CoreMigrationPlugin.java | 31 ++--
.../migration/DeDupConnectionDataMigration.java | 93 ++++++++++++
.../migration/EntityTypeMappingMigration.java | 104 --------------
.../corepersistence/migration/Versions.java | 31 ----
.../DeDupConnectionDataMigrationTest.java | 144 +++++++++++++++++++
.../migration/EntityTypeMappingMigrationIT.java | 125 ----------------
.../migration/data/TestProgressObserver.java | 10 +-
9 files changed, 314 insertions(+), 322 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/2689a015/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
index 7ed95e2..11707be 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
@@ -16,22 +16,43 @@
package org.apache.usergrid.corepersistence;
-import com.google.inject.AbstractModule;
-import com.google.inject.TypeLiteral;
-import com.google.inject.assistedinject.FactoryModuleBuilder;
-import com.google.inject.multibindings.Multibinder;
+import org.safehaus.guicyfig.GuicyFigModule;
+
import org.apache.usergrid.corepersistence.asyncevents.AsyncEventService;
import org.apache.usergrid.corepersistence.asyncevents.AsyncIndexProvider;
import org.apache.usergrid.corepersistence.asyncevents.EventBuilder;
import org.apache.usergrid.corepersistence.asyncevents.EventBuilderImpl;
-import org.apache.usergrid.corepersistence.index.*;
+import org.apache.usergrid.corepersistence.index.ApplicationIndexBucketLocator;
+import org.apache.usergrid.corepersistence.index.CoreIndexFig;
+import org.apache.usergrid.corepersistence.index.IndexLocationStrategyFactory;
+import org.apache.usergrid.corepersistence.index.IndexLocationStrategyFactoryImpl;
+import org.apache.usergrid.corepersistence.index.IndexProcessorFig;
+import org.apache.usergrid.corepersistence.index.IndexService;
+import org.apache.usergrid.corepersistence.index.IndexServiceImpl;
+import org.apache.usergrid.corepersistence.index.ReIndexService;
+import org.apache.usergrid.corepersistence.index.ReIndexServiceImpl;
import org.apache.usergrid.corepersistence.migration.CoreMigration;
import org.apache.usergrid.corepersistence.migration.CoreMigrationPlugin;
-import org.apache.usergrid.corepersistence.migration.EntityTypeMappingMigration;
+import org.apache.usergrid.corepersistence.migration.DeDupConnectionDataMigration;
import org.apache.usergrid.corepersistence.migration.MigrationModuleVersionPlugin;
import org.apache.usergrid.corepersistence.pipeline.PipelineModule;
-import org.apache.usergrid.corepersistence.rx.impl.*;
-import org.apache.usergrid.corepersistence.service.*;
+import org.apache.usergrid.corepersistence.rx.impl.AllApplicationsObservable;
+import org.apache.usergrid.corepersistence.rx.impl.AllApplicationsObservableImpl;
+import org.apache.usergrid.corepersistence.rx.impl.AllEntitiesInSystemImpl;
+import org.apache.usergrid.corepersistence.rx.impl.AllEntityIdsObservable;
+import org.apache.usergrid.corepersistence.rx.impl.AllEntityIdsObservableImpl;
+import org.apache.usergrid.corepersistence.rx.impl.AllNodesInGraphImpl;
+import org.apache.usergrid.corepersistence.service.AggregationService;
+import org.apache.usergrid.corepersistence.service.AggregationServiceFactory;
+import org.apache.usergrid.corepersistence.service.AggregationServiceImpl;
+import org.apache.usergrid.corepersistence.service.ApplicationService;
+import org.apache.usergrid.corepersistence.service.ApplicationServiceImpl;
+import org.apache.usergrid.corepersistence.service.CollectionService;
+import org.apache.usergrid.corepersistence.service.CollectionServiceImpl;
+import org.apache.usergrid.corepersistence.service.ConnectionService;
+import org.apache.usergrid.corepersistence.service.ConnectionServiceImpl;
+import org.apache.usergrid.corepersistence.service.StatusService;
+import org.apache.usergrid.corepersistence.service.StatusServiceImpl;
import org.apache.usergrid.persistence.collection.guice.CollectionModule;
import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
import org.apache.usergrid.persistence.core.guice.CommonModule;
@@ -42,7 +63,11 @@ import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.graph.guice.GraphModule;
import org.apache.usergrid.persistence.graph.serialization.impl.migration.GraphNode;
import org.apache.usergrid.persistence.index.guice.IndexModule;
-import org.safehaus.guicyfig.GuicyFigModule;
+
+import com.google.inject.AbstractModule;
+import com.google.inject.TypeLiteral;
+import com.google.inject.assistedinject.FactoryModuleBuilder;
+import com.google.inject.multibindings.Multibinder;
/**
@@ -105,7 +130,7 @@ public class CoreModule extends AbstractModule {
new TypeLiteral<DataMigration>() {}, CoreMigration.class );
- dataMigrationMultibinder.addBinding().to( EntityTypeMappingMigration.class );
+ dataMigrationMultibinder.addBinding().to( DeDupConnectionDataMigration.class );
//wire up the collection migration plugin
@@ -164,6 +189,14 @@ public class CoreModule extends AbstractModule {
bind( ApplicationService.class ).to( ApplicationServiceImpl.class );
bind( StatusService.class ).to( StatusServiceImpl.class );
+
+ /**
+ * Install migration services
+ */
+
+ //migrations
+ //we want to make sure our generics are retained, so we use a typeliteral
+
}
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/2689a015/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/CoreMigration.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/CoreMigration.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/CoreMigration.java
index 4df32da..b491e1f 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/CoreMigration.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/CoreMigration.java
@@ -1,40 +1,21 @@
/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
-package org.apache.usergrid.corepersistence.migration;/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
+package org.apache.usergrid.corepersistence.migration;
import java.lang.annotation.Retention;
http://git-wip-us.apache.org/repos/asf/usergrid/blob/2689a015/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/CoreMigrationPlugin.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/CoreMigrationPlugin.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/CoreMigrationPlugin.java
index 810f9fb..b69451a 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/CoreMigrationPlugin.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/CoreMigrationPlugin.java
@@ -1,20 +1,18 @@
/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
package org.apache.usergrid.corepersistence.migration;
@@ -22,9 +20,10 @@ package org.apache.usergrid.corepersistence.migration;
import java.util.Set;
-import org.apache.usergrid.persistence.core.migration.data.MigrationInfoSerialization;
+import org.apache.usergrid.persistence.collection.serialization.impl.migration.CollectionMigration;
import org.apache.usergrid.persistence.core.migration.data.AbstractMigrationPlugin;
import org.apache.usergrid.persistence.core.migration.data.DataMigration;
+import org.apache.usergrid.persistence.core.migration.data.MigrationInfoSerialization;
import org.apache.usergrid.persistence.core.migration.data.PluginPhase;
import com.google.inject.Inject;
@@ -37,7 +36,7 @@ import com.google.inject.Singleton;
@Singleton
public class CoreMigrationPlugin extends AbstractMigrationPlugin {
- public static final String PLUGIN_NAME = "core-data";
+ public static final String PLUGIN_NAME = "collections-entity-data";
http://git-wip-us.apache.org/repos/asf/usergrid/blob/2689a015/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/DeDupConnectionDataMigration.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/DeDupConnectionDataMigration.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/DeDupConnectionDataMigration.java
new file mode 100644
index 0000000..f65864a
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/DeDupConnectionDataMigration.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.usergrid.corepersistence.migration;
+
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.usergrid.corepersistence.rx.impl.AllApplicationsObservable;
+import org.apache.usergrid.corepersistence.service.ConnectionService;
+import org.apache.usergrid.persistence.core.migration.data.DataMigration;
+import org.apache.usergrid.persistence.core.migration.data.ProgressObserver;
+
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+
+
+@Singleton
+public class DeDupConnectionDataMigration implements DataMigration {
+
+ private static final Logger logger = LoggerFactory.getLogger(DeDupConnectionDataMigration.class);
+
+ private static final long UPDATE_COUNT = 1000;
+
+ private final ConnectionService connectionService;
+ private final AllApplicationsObservable allApplicationsObservable;
+
+
+ @Inject
+ public DeDupConnectionDataMigration( final ConnectionService connectionService,
+ final AllApplicationsObservable allApplicationsObservable ) {
+ this.connectionService = connectionService;
+ this.allApplicationsObservable = allApplicationsObservable;
+ }
+
+
+ @Override
+ public int migrate( final int currentVersion, final ProgressObserver observer ) {
+
+ final int migrationVersion = getMaxVersion();
+
+ observer.start();
+
+ connectionService.deDupeConnections( allApplicationsObservable.getData() ).reduce( 0l, (count, deDuped) ->{
+
+ final long newCount = count+1;
+
+ /**
+ * Update our progress observer
+ */
+ if(newCount % UPDATE_COUNT == 0){
+ logger.info( "De duped {} edges", newCount );
+ observer.update( migrationVersion, String.format("De duped %d edges", newCount) );
+ }
+
+ return newCount;
+
+ }).doOnNext( total -> {
+ logger.info("Completed de-duping {} edges", total );
+ observer.complete();;
+ }).subscribe();
+
+ return migrationVersion;
+
+ }
+
+
+ @Override
+ public boolean supports( final int currentVersion ) {
+ return currentVersion == getMaxVersion() - 1;
+ }
+
+
+ @Override
+ public int getMaxVersion() {
+ return 1;
+ }
+}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/2689a015/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/EntityTypeMappingMigration.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/EntityTypeMappingMigration.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/EntityTypeMappingMigration.java
deleted file mode 100644
index ef65ff2..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/EntityTypeMappingMigration.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.migration;
-
-
-import java.util.UUID;
-import java.util.concurrent.atomic.AtomicLong;
-
-import org.apache.usergrid.corepersistence.ManagerCache;
-import org.apache.usergrid.corepersistence.util.CpNamingUtils;
-import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
-import org.apache.usergrid.persistence.core.migration.data.DataMigration;
-import org.apache.usergrid.persistence.core.migration.data.MigrationDataProvider;
-import org.apache.usergrid.persistence.core.migration.data.ProgressObserver;
-import org.apache.usergrid.persistence.map.MapManager;
-import org.apache.usergrid.persistence.map.MapScope;
-
-import com.google.inject.Inject;
-
-import rx.Observable;
-import rx.schedulers.Schedulers;
-
-
-/**
- * Migration to ensure that our entity id is written into our map data
- */
-public class EntityTypeMappingMigration implements DataMigration {
-
- private final ManagerCache managerCache;
- private final MigrationDataProvider<EntityIdScope> allEntitiesInSystemObservable;
-
-
- @Inject
- public EntityTypeMappingMigration( final ManagerCache managerCache,
- final MigrationDataProvider<EntityIdScope> allEntitiesInSystemObservable ) {
- this.managerCache = managerCache;
- this.allEntitiesInSystemObservable = allEntitiesInSystemObservable;
- }
-
-
- @Override
- public int migrate( final int currentVersion,
- final ProgressObserver observer ) {
-
- final AtomicLong atomicLong = new AtomicLong();
-
-
- //migrate up to 100 types simultaneously
- allEntitiesInSystemObservable.getData().flatMap( entityIdScope -> {
- return Observable.just( entityIdScope ).doOnNext( entityIdScopeObservable -> {
- final MapScope ms = CpNamingUtils
- .getEntityTypeMapScope( entityIdScope.getApplicationScope().getApplication() );
-
- final MapManager mapManager = managerCache.getMapManager( ms );
-
- final UUID entityUuid = entityIdScope.getId().getUuid();
- final String entityType = entityIdScope.getId().getType();
-
- mapManager.putString( entityUuid.toString(), entityType );
-
- if ( atomicLong.incrementAndGet() % 100 == 0 ) {
- observer.update( getMaxVersion(),
- String.format( "Updated %d entities", atomicLong.get() ) );
- }
-
- } ).subscribeOn( Schedulers.io() );
- }, 100 ).count().toBlocking().last();
-
-
- return getMaxVersion();
-
-
- }
-
-
- @Override
- public boolean supports( final int currentVersion ) {
- //we move from the migration version fix to the current version
- return CoreDataVersions.INITIAL.getVersion() == currentVersion;
- }
-
-
- @Override
- public int getMaxVersion() {
- return CoreDataVersions.ID_MAP_FIX.getVersion();
- }
-}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/2689a015/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/Versions.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/Versions.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/Versions.java
deleted file mode 100644
index d921353..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/Versions.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- *
- * * Licensed to the Apache Software Foundation (ASF) under one
- * * or more contributor license agreements. See the NOTICE file
- * * distributed with this work for additional information
- * * regarding copyright ownership. The ASF licenses this file
- * * to you under the Apache License, Version 2.0 (the
- * * "License"); you may not use this file except in compliance
- * * with the License. You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing,
- * * software distributed under the License is distributed on an
- * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * * KIND, either express or implied. See the License for the
- * * specific language governing permissions and limitations
- * * under the License.
- *
- */
-
-package org.apache.usergrid.corepersistence.migration;
-
-
-/**
- * Simple class to hold the constants of all versions
- */
-public class Versions {
-
-
-}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/2689a015/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/DeDupConnectionDataMigrationTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/DeDupConnectionDataMigrationTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/DeDupConnectionDataMigrationTest.java
new file mode 100644
index 0000000..c84ad2b
--- /dev/null
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/DeDupConnectionDataMigrationTest.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.usergrid.corepersistence.migration;
+
+
+import java.util.List;
+
+import org.junit.Test;
+
+import org.apache.usergrid.corepersistence.rx.impl.AllApplicationsObservable;
+import org.apache.usergrid.corepersistence.service.ConnectionScope;
+import org.apache.usergrid.corepersistence.service.ConnectionService;
+import org.apache.usergrid.corepersistence.util.CpNamingUtils;
+import org.apache.usergrid.persistence.core.migration.data.TestProgressObserver;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
+import org.apache.usergrid.persistence.graph.Edge;
+
+import rx.Observable;
+import rx.Subscriber;
+
+import static org.apache.usergrid.persistence.core.util.IdGenerator.createId;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+
+public class DeDupConnectionDataMigrationTest {
+
+ @Test
+ public void testVersion() {
+
+ //mock up an initial system state
+ final int version = 0;
+
+ final ConnectionService connectionService = mock( ConnectionService.class );
+
+ final AllApplicationsObservable allApplicationsObservable = mock( AllApplicationsObservable.class );
+
+
+ final DeDupConnectionDataMigration plugin =
+ new DeDupConnectionDataMigration( connectionService, allApplicationsObservable );
+
+
+ assertTrue( plugin.supports( version ) );
+
+ assertFalse( plugin.supports( plugin.getMaxVersion() ) );
+ }
+
+
+ /**
+ * Test
+ */
+ @Test
+ public void testEmitEdges() {
+
+ //mock up an initial system state
+ final int version = 0;
+
+ final ConnectionService connectionService = mock( ConnectionService.class );
+
+
+ final AllApplicationsObservable allApplicationsObservable = mock( AllApplicationsObservable.class );
+
+
+ final int count = 3000;
+
+ final Observable<ConnectionScope> edgeEmitter = Observable.create( new DupConnectionEmitter( count ) );
+
+ when( connectionService.deDupeConnections( any( Observable.class ) ) ).thenReturn( edgeEmitter );
+
+
+ final DeDupConnectionDataMigration plugin =
+ new DeDupConnectionDataMigration( connectionService, allApplicationsObservable );
+
+
+ final TestProgressObserver testProgressObserver = new TestProgressObserver();
+
+
+ plugin.migrate( version, testProgressObserver );
+
+ final List<String> updates = testProgressObserver.getUpdates();
+
+ assertEquals( "Expected 3 updates", 3, updates.size() );
+
+
+ assertEquals( "De duped 1000 edges", updates.get( 0 ) );
+ assertEquals( "De duped 2000 edges", updates.get( 1 ) );
+ assertEquals( "De duped 3000 edges", updates.get( 2 ) );
+
+
+ assertTrue( "Should complete", testProgressObserver.isComplete() );
+ assertTrue( "Should start", testProgressObserver.isStarted() );
+ assertFalse( "Should not fail", testProgressObserver.isFailed() );
+ }
+
+
+ private final class DupConnectionEmitter implements Observable.OnSubscribe<ConnectionScope> {
+
+
+ private final int count;
+
+
+ private DupConnectionEmitter( final int count ) {this.count = count;}
+
+
+ @Override
+ public void call( final Subscriber<? super ConnectionScope> subscriber ) {
+
+ final ApplicationScope applicationScope = new ApplicationScopeImpl( createId( "application" ) );
+ final Edge edge = CpNamingUtils.createConnectionEdge( createId( "source" ), "test", createId( "target" ) );
+
+
+ final ConnectionScope scope = new ConnectionScope( applicationScope, edge );
+
+
+ subscriber.onStart();
+
+ for ( int i = 0; i < count; i++ ) {
+ subscriber.onNext( scope );
+ }
+
+ subscriber.onCompleted();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/2689a015/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/EntityTypeMappingMigrationIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/EntityTypeMappingMigrationIT.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/EntityTypeMappingMigrationIT.java
deleted file mode 100644
index 7d22abe..0000000
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/EntityTypeMappingMigrationIT.java
+++ /dev/null
@@ -1,125 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.migration;
-
-
-import org.junit.Test;
-
-import org.apache.usergrid.AbstractCoreIT;
-import org.apache.usergrid.corepersistence.ManagerCache;
-import org.apache.usergrid.corepersistence.util.CpNamingUtils;
-import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
-import org.apache.usergrid.persistence.core.migration.data.TestMigrationDataProvider;
-import org.apache.usergrid.persistence.core.migration.data.TestProgressObserver;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
-import org.apache.usergrid.persistence.map.MapManager;
-import org.apache.usergrid.persistence.map.MapScope;
-import org.apache.usergrid.persistence.map.impl.MapScopeImpl;
-import org.apache.usergrid.persistence.model.entity.Id;
-
-import rx.Observable;
-
-import static org.apache.usergrid.persistence.core.util.IdGenerator.createId;
-import static org.junit.Assert.assertEquals;
-import static org.mockito.Matchers.eq;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
-
-
-/**
- * Test for our entity type mapping
- */
-public class EntityTypeMappingMigrationIT {
-
-
- @Test
- public void testIdMapping() throws Throwable {
-
- final Id applicationId = createId("application");
-
- final ApplicationScope scope1 = new ApplicationScopeImpl( applicationId );
-
- final Id entityId1 = createId("thing");
-
- final EntityIdScope idScope1 = new EntityIdScope(scope1, entityId1 );
-
- final MapScope mapScope1 = new MapScopeImpl(applicationId, CpNamingUtils.TYPES_BY_UUID_MAP );
-
-
-
- final ApplicationScope scope2 = new ApplicationScopeImpl( applicationId);
-
- final Id entityId2 = createId("foo");
-
- final EntityIdScope idScope2 = new EntityIdScope( scope2, entityId2 );
-
- final MapScope mapScope2 = new MapScopeImpl(applicationId, CpNamingUtils.TYPES_BY_UUID_MAP );
-
-
- final Observable<EntityIdScope> scopes = Observable.just(idScope1, idScope2);
-
- final TestMigrationDataProvider<EntityIdScope> migrationDataProvider = new TestMigrationDataProvider<>();
-
- //set our scopes
- migrationDataProvider.setObservable( scopes );
-
-
-
-
- //mock up returning our map manager
- final MapManager mapManager = mock(MapManager.class);
- final ManagerCache managerCache = mock(ManagerCache.class);
-
- when(managerCache.getMapManager( eq( mapScope1 ) )).thenReturn( mapManager );
-
- when(managerCache.getMapManager( eq( mapScope2 ) )).thenReturn( mapManager );
-
- final TestProgressObserver progressObserver = new TestProgressObserver();
-
-
- //wire it up
- final EntityTypeMappingMigration migration = new EntityTypeMappingMigration( managerCache, migrationDataProvider );
-
- //run it
-
- final int returnedVersion = migration.migrate(CoreDataVersions.INITIAL.getVersion(), progressObserver );
-
-
- assertEquals(CoreDataVersions.ID_MAP_FIX.getVersion(), returnedVersion);
-
- //verify we saved it
-
- verify(mapManager).putString(entityId1.getUuid().toString(), entityId1.getType() );
-
- verify(mapManager).putString(entityId2.getUuid().toString(), entityId2.getType() );
-
-
-
-
-
-
-
-
-
-
- }
-}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/2689a015/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/TestProgressObserver.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/TestProgressObserver.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/TestProgressObserver.java
index 7f59151..21d6013 100644
--- a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/TestProgressObserver.java
+++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/TestProgressObserver.java
@@ -30,19 +30,21 @@ public class TestProgressObserver implements ProgressObserver {
private boolean started = false;
+ private boolean complete = false;
+
private List<String> updates = new ArrayList<>( 100 );
@Override
public void start() {
- started = true;
+ started = true;
}
@Override
public void complete() {
- started = false;
+ complete = true;
}
@@ -66,7 +68,6 @@ public class TestProgressObserver implements ProgressObserver {
/**
* Get if we failed
- * @return
*/
public boolean isFailed() {
return failed;
@@ -78,10 +79,11 @@ public class TestProgressObserver implements ProgressObserver {
}
+ public boolean isComplete() { return complete;}
+
/**
* Get update messages
- * @return
*/
public List<String> getUpdates() {
return updates;
[07/13] usergrid git commit: Refactored migrations to read their own
data sets rather than be passed with an invocation
Posted by to...@apache.org.
Refactored migrations to read their own data sets rather than be passed with an invocation
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/d971ec50
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/d971ec50
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/d971ec50
Branch: refs/heads/two-dot-o-dev
Commit: d971ec509528fa24977a15501776ebb356e6e1c7
Parents: fb1cf2e
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Sep 16 16:44:02 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Sep 16 16:44:02 2015 -0600
----------------------------------------------------------------------
.../usergrid/corepersistence/CoreModule.java | 4 +-
.../migration/CoreMigrationPlugin.java | 9 +--
.../migration/EntityTypeMappingMigration.java | 4 +-
.../migration/MigrationModuleVersionPlugin.java | 5 --
.../migration/EntityTypeMappingMigrationIT.java | 2 +-
.../serialization/impl/SerializationModule.java | 4 +-
.../migration/CollectionMigrationPlugin.java | 8 +--
.../migration/MvccEntityDataMigrationImpl.java | 10 ++--
...ctMvccEntityDataMigrationV1ToV3ImplTest.java | 9 +--
.../migration/data/AbstractMigrationPlugin.java | 17 +++---
.../core/migration/data/DataMigration.java | 11 ++--
.../migration/data/MigrationDataProvider.java | 2 +-
.../core/migration/data/MigrationPlugin.java | 4 +-
.../persistence/graph/guice/GraphModule.java | 4 +-
.../impl/migration/EdgeDataMigrationImpl.java | 12 ++--
.../impl/migration/GraphMigrationPlugin.java | 8 +--
.../migration/EdgeDataMigrationImplTest.java | 59 +++++++++++---------
.../persistence/index/guice/IndexModule.java | 4 +-
.../index/migration/EsIndexMigrationPlugin.java | 8 +--
19 files changed, 86 insertions(+), 98 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/d971ec50/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
index e290cfe..7ed95e2 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
@@ -100,9 +100,9 @@ public class CoreModule extends AbstractModule {
/**
* Create our migrations for within our core plugin
*/
- Multibinder<DataMigration<EntityIdScope>> dataMigrationMultibinder =
+ Multibinder<DataMigration> dataMigrationMultibinder =
Multibinder.newSetBinder( binder(),
- new TypeLiteral<DataMigration<EntityIdScope>>() {}, CoreMigration.class );
+ new TypeLiteral<DataMigration>() {}, CoreMigration.class );
dataMigrationMultibinder.addBinding().to( EntityTypeMappingMigration.class );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/d971ec50/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/CoreMigrationPlugin.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/CoreMigrationPlugin.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/CoreMigrationPlugin.java
index 58233e0..810f9fb 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/CoreMigrationPlugin.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/CoreMigrationPlugin.java
@@ -22,11 +22,9 @@ package org.apache.usergrid.corepersistence.migration;
import java.util.Set;
-import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
import org.apache.usergrid.persistence.core.migration.data.MigrationInfoSerialization;
import org.apache.usergrid.persistence.core.migration.data.AbstractMigrationPlugin;
import org.apache.usergrid.persistence.core.migration.data.DataMigration;
-import org.apache.usergrid.persistence.core.migration.data.MigrationDataProvider;
import org.apache.usergrid.persistence.core.migration.data.PluginPhase;
import com.google.inject.Inject;
@@ -37,17 +35,16 @@ import com.google.inject.Singleton;
* Migration plugin for the collection module
*/
@Singleton
-public class CoreMigrationPlugin extends AbstractMigrationPlugin<EntityIdScope> {
+public class CoreMigrationPlugin extends AbstractMigrationPlugin {
public static final String PLUGIN_NAME = "core-data";
@Inject
- public CoreMigrationPlugin( @CoreMigration final Set<DataMigration<EntityIdScope>> entityDataMigrations,
- final MigrationDataProvider<EntityIdScope> entityIdScopeDataMigrationProvider,
+ public CoreMigrationPlugin( @CoreMigration final Set<DataMigration> entityDataMigrations,
final MigrationInfoSerialization migrationInfoSerialization ) {
- super( entityDataMigrations, entityIdScopeDataMigrationProvider, migrationInfoSerialization );
+ super( entityDataMigrations, migrationInfoSerialization );
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/d971ec50/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/EntityTypeMappingMigration.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/EntityTypeMappingMigration.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/EntityTypeMappingMigration.java
index 47ad30a..ef65ff2 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/EntityTypeMappingMigration.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/EntityTypeMappingMigration.java
@@ -41,7 +41,7 @@ import rx.schedulers.Schedulers;
/**
* Migration to ensure that our entity id is written into our map data
*/
-public class EntityTypeMappingMigration implements DataMigration<EntityIdScope> {
+public class EntityTypeMappingMigration implements DataMigration {
private final ManagerCache managerCache;
private final MigrationDataProvider<EntityIdScope> allEntitiesInSystemObservable;
@@ -56,7 +56,7 @@ public class EntityTypeMappingMigration implements DataMigration<EntityIdScope>
@Override
- public int migrate( final int currentVersion, final MigrationDataProvider<EntityIdScope> migrationDataProvider,
+ public int migrate( final int currentVersion,
final ProgressObserver observer ) {
final AtomicLong atomicLong = new AtomicLong();
http://git-wip-us.apache.org/repos/asf/usergrid/blob/d971ec50/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/MigrationModuleVersionPlugin.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/MigrationModuleVersionPlugin.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/MigrationModuleVersionPlugin.java
index 2ffe335..145e76f 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/MigrationModuleVersionPlugin.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/MigrationModuleVersionPlugin.java
@@ -56,11 +56,6 @@ public class MigrationModuleVersionPlugin implements MigrationPlugin{
*/
private static final int ENTITY_V2_MIGRATION = 3;
- /**
- * Appinfo to application_info migration.
- */
- private static final int APPINFO_MIGRATION = 4;
-
/**
* Get versions directly from impls so we know they're accurate
http://git-wip-us.apache.org/repos/asf/usergrid/blob/d971ec50/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/EntityTypeMappingMigrationIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/EntityTypeMappingMigrationIT.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/EntityTypeMappingMigrationIT.java
index 84215a1..7d22abe 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/EntityTypeMappingMigrationIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/EntityTypeMappingMigrationIT.java
@@ -101,7 +101,7 @@ public class EntityTypeMappingMigrationIT {
//run it
- final int returnedVersion = migration.migrate(CoreDataVersions.INITIAL.getVersion(), migrationDataProvider, progressObserver );
+ final int returnedVersion = migration.migrate(CoreDataVersions.INITIAL.getVersion(), progressObserver );
assertEquals(CoreDataVersions.ID_MAP_FIX.getVersion(), returnedVersion);
http://git-wip-us.apache.org/repos/asf/usergrid/blob/d971ec50/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/SerializationModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/SerializationModule.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/SerializationModule.java
index bd2728b..3a6fd14 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/SerializationModule.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/SerializationModule.java
@@ -96,8 +96,8 @@ public class SerializationModule extends AbstractModule {
//migrations
//we want to make sure our generics are retained, so we use a typeliteral
- Multibinder<DataMigration<EntityIdScope>> dataMigrationMultibinder = Multibinder
- .newSetBinder( binder(), new TypeLiteral<DataMigration<EntityIdScope>>() {}, CollectionMigration.class );
+ Multibinder<DataMigration> dataMigrationMultibinder = Multibinder
+ .newSetBinder( binder(), new TypeLiteral<DataMigration>() {}, CollectionMigration.class );
dataMigrationMultibinder.addBinding().to( MvccEntityDataMigrationImpl.class );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/d971ec50/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/CollectionMigrationPlugin.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/CollectionMigrationPlugin.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/CollectionMigrationPlugin.java
index c757d28..4e92d3c 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/CollectionMigrationPlugin.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/CollectionMigrationPlugin.java
@@ -28,7 +28,6 @@ import java.util.Set;
import org.apache.usergrid.persistence.core.migration.data.AbstractMigrationPlugin;
import org.apache.usergrid.persistence.core.migration.data.DataMigration;
-import org.apache.usergrid.persistence.core.migration.data.MigrationDataProvider;
import org.apache.usergrid.persistence.core.migration.data.MigrationInfoSerialization;
import org.apache.usergrid.persistence.core.migration.data.PluginPhase;
@@ -40,7 +39,7 @@ import com.google.inject.Singleton;
* Migration plugin for the collection module
*/
@Singleton
-public class CollectionMigrationPlugin extends AbstractMigrationPlugin<EntityIdScope> {
+public class CollectionMigrationPlugin extends AbstractMigrationPlugin {
public static final String PLUGIN_NAME = "collections-entity-data";
@@ -48,10 +47,9 @@ public class CollectionMigrationPlugin extends AbstractMigrationPlugin<EntityIdS
@Inject
public CollectionMigrationPlugin( @CollectionMigration
- final Set<DataMigration<EntityIdScope>> entityDataMigrations,
- final MigrationDataProvider<EntityIdScope> entityIdScopeDataMigrationProvider,
+ final Set<DataMigration> entityDataMigrations,
final MigrationInfoSerialization migrationInfoSerialization ) {
- super( entityDataMigrations, entityIdScopeDataMigrationProvider, migrationInfoSerialization );
+ super( entityDataMigrations, migrationInfoSerialization );
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/d971ec50/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/MvccEntityDataMigrationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/MvccEntityDataMigrationImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/MvccEntityDataMigrationImpl.java
index 86fd701..5143c49 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/MvccEntityDataMigrationImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/MvccEntityDataMigrationImpl.java
@@ -65,7 +65,7 @@ import rx.schedulers.Schedulers;
* Data migration strategy for entities
*/
@Singleton
-public class MvccEntityDataMigrationImpl implements DataMigration<EntityIdScope> {
+public class MvccEntityDataMigrationImpl implements DataMigration{
private static final Logger LOGGER = LoggerFactory.getLogger( MvccEntityDataMigrationImpl.class );
@@ -75,6 +75,7 @@ public class MvccEntityDataMigrationImpl implements DataMigration<EntityIdScope>
private final MvccEntitySerializationStrategyV3Impl mvccEntitySerializationStrategyV3;
private final UniqueValueSerializationStrategy uniqueValueSerializationStrategy;
private final MvccLogEntrySerializationStrategy mvccLogEntrySerializationStrategy;
+ private final MigrationDataProvider<EntityIdScope> migrationDataProvider;
@Inject
@@ -82,12 +83,14 @@ public class MvccEntityDataMigrationImpl implements DataMigration<EntityIdScope>
final VersionedMigrationSet<MvccEntitySerializationStrategy> allVersions,
final MvccEntitySerializationStrategyV3Impl mvccEntitySerializationStrategyV3,
final UniqueValueSerializationStrategy uniqueValueSerializationStrategy,
- final MvccLogEntrySerializationStrategy mvccLogEntrySerializationStrategy ) {
+ final MvccLogEntrySerializationStrategy mvccLogEntrySerializationStrategy,
+ final MigrationDataProvider<EntityIdScope> migrationDataProvider ) {
this.keyspace = keyspace;
this.allVersions = allVersions;
this.mvccEntitySerializationStrategyV3 = mvccEntitySerializationStrategyV3;
this.uniqueValueSerializationStrategy = uniqueValueSerializationStrategy;
this.mvccLogEntrySerializationStrategy = mvccLogEntrySerializationStrategy;
+ this.migrationDataProvider = migrationDataProvider;
}
@@ -105,8 +108,7 @@ public class MvccEntityDataMigrationImpl implements DataMigration<EntityIdScope>
@Override
- public int migrate( final int currentVersion, final MigrationDataProvider<EntityIdScope> migrationDataProvider,
- final ProgressObserver observer ) {
+ public int migrate( final int currentVersion, final ProgressObserver observer ) {
final AtomicLong atomicLong = new AtomicLong();
http://git-wip-us.apache.org/repos/asf/usergrid/blob/d971ec50/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/AbstractMvccEntityDataMigrationV1ToV3ImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/AbstractMvccEntityDataMigrationV1ToV3ImplTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/AbstractMvccEntityDataMigrationV1ToV3ImplTest.java
index 986b740..f51b339 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/AbstractMvccEntityDataMigrationV1ToV3ImplTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/AbstractMvccEntityDataMigrationV1ToV3ImplTest.java
@@ -121,13 +121,6 @@ public abstract class AbstractMvccEntityDataMigrationV1ToV3ImplTest implements D
Observable.just( new EntityIdScope( scope, entity1.getId() ), new EntityIdScope( scope, entity2.getId() ) );
- final MigrationDataProvider<EntityIdScope> migrationProvider = new MigrationDataProvider<EntityIdScope>() {
- @Override
- public Observable<EntityIdScope> getData() {
- return entityIdScope;
- }
- };
-
final TestProgressObserver progressObserver = new TestProgressObserver();
final CollectionDataVersions startVersion = getSourceVersion();
@@ -141,7 +134,7 @@ public abstract class AbstractMvccEntityDataMigrationV1ToV3ImplTest implements D
//now migration
final int newVersion = mvccEntityDataMigrationImpl
- .migrate( startVersion.getVersion(), migrationProvider, progressObserver );
+ .migrate( startVersion.getVersion(), progressObserver );
final CollectionDataVersions expectedVersion = expectedTargetVersion();
http://git-wip-us.apache.org/repos/asf/usergrid/blob/d971ec50/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/AbstractMigrationPlugin.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/AbstractMigrationPlugin.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/AbstractMigrationPlugin.java
index 0cd26d4..9671bc5 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/AbstractMigrationPlugin.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/AbstractMigrationPlugin.java
@@ -34,23 +34,20 @@ import org.slf4j.LoggerFactory;
* Standard implementation logic for plugins to extend
* @param <T>
*/
-public abstract class AbstractMigrationPlugin<T> implements MigrationPlugin {
+public abstract class AbstractMigrationPlugin implements MigrationPlugin {
private static final Logger LOG = LoggerFactory.getLogger( AbstractMigrationPlugin.class );
- private final Set<DataMigration<T>> entityDataMigrations;
- private final MigrationDataProvider<T> entityIdScopeDataMigrationProvider;
+ private final Set<DataMigration> entityDataMigrations;
private final MigrationInfoSerialization migrationInfoSerialization;
- protected AbstractMigrationPlugin( final Set<DataMigration<T>> entityDataMigrations,
- final MigrationDataProvider<T> entityIdScopeDataMigrationProvider,
+ protected AbstractMigrationPlugin( final Set<DataMigration>entityDataMigrations,
final MigrationInfoSerialization migrationInfoSerialization ) {
this.entityDataMigrations = entityDataMigrations;
- this.entityIdScopeDataMigrationProvider = entityIdScopeDataMigrationProvider;
this.migrationInfoSerialization = migrationInfoSerialization;
}
@@ -71,7 +68,7 @@ public abstract class AbstractMigrationPlugin<T> implements MigrationPlugin {
int max = 0;
- for(DataMigration<T> entityMigration: entityDataMigrations){
+ for(DataMigration entityMigration: entityDataMigrations){
max = Math.max( max, entityMigration.getMaxVersion() );
}
@@ -85,12 +82,12 @@ public abstract class AbstractMigrationPlugin<T> implements MigrationPlugin {
* @return True if we ran a migration
*/
private boolean runMigration( final ProgressObserver po ) {
- DataMigration<T> migrationToExecute = null;
+ DataMigration migrationToExecute = null;
final int version = migrationInfoSerialization.getVersion( getName() );
- for ( DataMigration<T> entityMigration : entityDataMigrations ) {
+ for ( DataMigration entityMigration : entityDataMigrations ) {
if ( entityMigration.supports( version ) ) {
if ( migrationToExecute != null ) {
throw new DataMigrationException(
@@ -113,7 +110,7 @@ public abstract class AbstractMigrationPlugin<T> implements MigrationPlugin {
//run the migration
- final int newSystemVersion = migrationToExecute.migrate( version, entityIdScopeDataMigrationProvider, po );
+ final int newSystemVersion = migrationToExecute.migrate( version, po );
//write the version
migrationInfoSerialization.setVersion( getName(), newSystemVersion );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/d971ec50/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/DataMigration.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/DataMigration.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/DataMigration.java
index ce250cc..a3ee675 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/DataMigration.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/DataMigration.java
@@ -27,32 +27,31 @@ package org.apache.usergrid.persistence.core.migration.data;
/**
* Data migration. The internal version to migrate
*
- * @param <T>
+ * @param
*/
-public interface DataMigration<T> {
+public interface DataMigration {
/**
* Perform the migration, returning an observable with a single emitted value
* @param currentVersion the current version of the system
- * @param migrationDataProvider
* @param observer The observer to receive updates of the progress
*
* @return The version that the system is now running
*/
- public int migrate(final int currentVersion, MigrationDataProvider<T> migrationDataProvider, ProgressObserver observer);
+ int migrate( final int currentVersion, ProgressObserver observer );
/**
* Check if this version supports migration from the current system version. If this returns false,
* migrate will not be invoked
* @return
*/
- public boolean supports(final int currentVersion);
+ boolean supports( final int currentVersion );
/**
* Get the max version this migration can migrate to
* @return
*/
- public int getMaxVersion();
+ int getMaxVersion();
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/d971ec50/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/MigrationDataProvider.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/MigrationDataProvider.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/MigrationDataProvider.java
index e232d7b..1608742 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/MigrationDataProvider.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/MigrationDataProvider.java
@@ -38,6 +38,6 @@ public interface MigrationDataProvider<T> {
* Get data that can be used in the migration
* @return
*/
- public Observable<T> getData();
+ Observable<T> getData();
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/d971ec50/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/MigrationPlugin.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/MigrationPlugin.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/MigrationPlugin.java
index 4953f25..cb9297d 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/MigrationPlugin.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/MigrationPlugin.java
@@ -34,12 +34,12 @@ public interface MigrationPlugin {
* Get the name of the plugin. Must be unique
* @return
*/
- public String getName();
+ String getName();
/**
* Run any migrations that may need to be run in this plugin
*/
- public void run(ProgressObserver observer);
+ void run( ProgressObserver observer );
/**
http://git-wip-us.apache.org/repos/asf/usergrid/blob/d971ec50/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/guice/GraphModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/guice/GraphModule.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/guice/GraphModule.java
index 4b628d1..6280c7c 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/guice/GraphModule.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/guice/GraphModule.java
@@ -128,8 +128,8 @@ public abstract class GraphModule extends AbstractModule {
//wire up the edg migration
- Multibinder<DataMigration<GraphNode>> dataMigrationMultibinder =
- Multibinder.newSetBinder( binder(), new TypeLiteral<DataMigration<GraphNode>>() {}, GraphMigration.class );
+ Multibinder<DataMigration> dataMigrationMultibinder =
+ Multibinder.newSetBinder( binder(), new TypeLiteral<DataMigration>() {}, GraphMigration.class );
dataMigrationMultibinder.addBinding().to( EdgeDataMigrationImpl.class );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/d971ec50/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/EdgeDataMigrationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/EdgeDataMigrationImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/EdgeDataMigrationImpl.java
index d6c42e3..15ff337 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/EdgeDataMigrationImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/EdgeDataMigrationImpl.java
@@ -49,7 +49,7 @@ import java.util.concurrent.atomic.AtomicLong;
* The migration data provider assumes that is will visit every node in the graph
* all edges from these source node will then be re-indexed.
*/
-public class EdgeDataMigrationImpl implements DataMigration<GraphNode> {
+public class EdgeDataMigrationImpl implements DataMigration {
private static final Logger logger = LoggerFactory.getLogger(EdgeDataMigrationImpl.class);
@@ -58,24 +58,26 @@ public class EdgeDataMigrationImpl implements DataMigration<GraphNode> {
private final EdgesObservable edgesFromSourceObservable;
private final VersionedMigrationSet<EdgeMetadataSerialization> allVersions;
private final EdgeMetadataSerializationV2Impl edgeMetadataSerializationV2;
+ private final MigrationDataProvider<GraphNode> migrationDataProvider;
@Inject
- public EdgeDataMigrationImpl( final Keyspace keyspace,
- final GraphManagerFactory graphManagerFactory,
+ public EdgeDataMigrationImpl( final Keyspace keyspace, final GraphManagerFactory graphManagerFactory,
final EdgesObservable edgesFromSourceObservable,
final VersionedMigrationSet<EdgeMetadataSerialization> allVersions,
- final EdgeMetadataSerializationV2Impl edgeMetadataSerializationV2 ) {
+ final EdgeMetadataSerializationV2Impl edgeMetadataSerializationV2,
+ final MigrationDataProvider<GraphNode> migrationDataProvider ) {
this.keyspace = keyspace;
this.graphManagerFactory = graphManagerFactory;
this.edgesFromSourceObservable = edgesFromSourceObservable;
this.allVersions = allVersions;
this.edgeMetadataSerializationV2 = edgeMetadataSerializationV2;
+ this.migrationDataProvider = migrationDataProvider;
}
@Override
- public int migrate( final int currentVersion, final MigrationDataProvider<GraphNode> migrationDataProvider,
+ public int migrate( final int currentVersion,
final ProgressObserver observer ) {
final AtomicLong counter = new AtomicLong();
http://git-wip-us.apache.org/repos/asf/usergrid/blob/d971ec50/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/GraphMigrationPlugin.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/GraphMigrationPlugin.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/GraphMigrationPlugin.java
index ed51c27..97edf3b 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/GraphMigrationPlugin.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/GraphMigrationPlugin.java
@@ -29,7 +29,6 @@ import java.util.Set;
import org.apache.usergrid.persistence.core.migration.data.MigrationInfoSerialization;
import org.apache.usergrid.persistence.core.migration.data.AbstractMigrationPlugin;
import org.apache.usergrid.persistence.core.migration.data.DataMigration;
-import org.apache.usergrid.persistence.core.migration.data.MigrationDataProvider;
import org.apache.usergrid.persistence.core.migration.data.PluginPhase;
import com.google.inject.Inject;
@@ -41,7 +40,7 @@ import com.google.inject.Singleton;
* Migration plugin for the collection module
*/
@Singleton
-public class GraphMigrationPlugin extends AbstractMigrationPlugin<GraphNode> {
+public class GraphMigrationPlugin extends AbstractMigrationPlugin {
public static final String PLUGIN_NAME = "graph-data";
@@ -49,10 +48,9 @@ public class GraphMigrationPlugin extends AbstractMigrationPlugin<GraphNode> {
@Inject
public GraphMigrationPlugin(@GraphMigration
- final Set<DataMigration<GraphNode>> entityDataMigrations,
- final MigrationDataProvider<GraphNode> entityIdScopeDataMigrationProvider,
+ final Set<DataMigration> entityDataMigrations,
final MigrationInfoSerialization migrationInfoSerialization ) {
- super( entityDataMigrations, entityIdScopeDataMigrationProvider, migrationInfoSerialization );
+ super( entityDataMigrations, migrationInfoSerialization );
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/d971ec50/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/EdgeDataMigrationImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/EdgeDataMigrationImplTest.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/EdgeDataMigrationImplTest.java
index 049c3d2..bc2afdc 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/EdgeDataMigrationImplTest.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/EdgeDataMigrationImplTest.java
@@ -28,8 +28,9 @@ import org.junit.runner.RunWith;
import org.apache.usergrid.persistence.core.guice.DataMigrationResetRule;
import org.apache.usergrid.persistence.core.migration.data.DataMigrationManager;
-import org.apache.usergrid.persistence.core.migration.data.TestProgressObserver;
import org.apache.usergrid.persistence.core.migration.data.MigrationDataProvider;
+import org.apache.usergrid.persistence.core.migration.data.TestProgressObserver;
+import org.apache.usergrid.persistence.core.migration.data.VersionedMigrationSet;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
import org.apache.usergrid.persistence.core.test.ITRunner;
@@ -40,25 +41,28 @@ import org.apache.usergrid.persistence.graph.GraphManager;
import org.apache.usergrid.persistence.graph.GraphManagerFactory;
import org.apache.usergrid.persistence.graph.guice.TestGraphModule;
import org.apache.usergrid.persistence.graph.impl.SimpleSearchEdgeType;
+import org.apache.usergrid.persistence.graph.serialization.EdgeMetadataSerialization;
+import org.apache.usergrid.persistence.graph.serialization.EdgesObservable;
+import org.apache.usergrid.persistence.graph.serialization.impl.EdgeMetadataSerializationV2Impl;
import org.apache.usergrid.persistence.graph.serialization.impl.GraphDataVersions;
import org.apache.usergrid.persistence.model.entity.Id;
import com.google.inject.Inject;
+import com.netflix.astyanax.Keyspace;
import net.jcip.annotations.NotThreadSafe;
import rx.Observable;
import static org.apache.usergrid.persistence.graph.test.util.EdgeTestUtils.createEdge;
-import static org.apache.usergrid.persistence.core.util.IdGenerator.createId;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
@NotThreadSafe
-@RunWith(ITRunner.class)
-@UseModules({ TestGraphModule.class })
+@RunWith( ITRunner.class )
+@UseModules( { TestGraphModule.class } )
public class EdgeDataMigrationImplTest implements DataMigrationResetRule.DataMigrationManagerProvider {
@@ -68,16 +72,26 @@ public class EdgeDataMigrationImplTest implements DataMigrationResetRule.DataMig
@Inject
public GraphManagerFactory graphManagerFactory;
+
+ @Inject
+ public Keyspace keyspace;
+
@Inject
- public EdgeDataMigrationImpl edgeDataMigrationImpl;
+ public EdgesObservable edgesFromSourceObservable;
+
+ @Inject
+ public VersionedMigrationSet<EdgeMetadataSerialization> allVersions;
+
+ @Inject
+ public EdgeMetadataSerializationV2Impl edgeMetadataSerializationV2;
+
/**
* Rule to do the resets we need
*/
@Rule
- public DataMigrationResetRule migrationTestRule = new DataMigrationResetRule(this, GraphMigrationPlugin.PLUGIN_NAME, GraphDataVersions.INITIAL.getVersion());
-
-
+ public DataMigrationResetRule migrationTestRule =
+ new DataMigrationResetRule( this, GraphMigrationPlugin.PLUGIN_NAME, GraphDataVersions.INITIAL.getVersion() );
@Test
@@ -90,7 +104,7 @@ public class EdgeDataMigrationImplTest implements DataMigrationResetRule.DataMig
final ApplicationScope applicationScope = new ApplicationScopeImpl( IdGenerator.createId( "application" ) );
- GraphManager gm = graphManagerFactory.createEdgeManager(applicationScope );
+ GraphManager gm = graphManagerFactory.createEdgeManager( applicationScope );
final Id sourceId1 = IdGenerator.createId( "source1" );
@@ -103,17 +117,15 @@ public class EdgeDataMigrationImplTest implements DataMigrationResetRule.DataMig
final Id target2 = IdGenerator.createId( "target2" );
- Edge s1t1 = createEdge(sourceId1, "test", target1 );
+ Edge s1t1 = createEdge( sourceId1, "test", target1 );
Edge s1t2 = createEdge( sourceId1, "baz", target2 );
- Edge s2t1 = createEdge(sourceId2, "foo", target1);
+ Edge s2t1 = createEdge( sourceId2, "foo", target1 );
Edge s2t2 = createEdge( sourceId2, "bar", target2 );
-
-
gm.writeEdge( s1t1 ).toBlocking().last();
gm.writeEdge( s1t2 ).toBlocking().last();
gm.writeEdge( s2t1 ).toBlocking().last();
@@ -121,24 +133,24 @@ public class EdgeDataMigrationImplTest implements DataMigrationResetRule.DataMig
//walk from s1 and s2
- final Observable<GraphNode> graphNodes = Observable.just( new GraphNode( applicationScope, sourceId1), new GraphNode(applicationScope, sourceId2 ) );
+ final Observable<GraphNode> graphNodes = Observable
+ .just( new GraphNode( applicationScope, sourceId1 ), new GraphNode( applicationScope, sourceId2 ) );
- final MigrationDataProvider<GraphNode> testMigrationProvider = new MigrationDataProvider<GraphNode>() {
- @Override
- public Observable<GraphNode> getData() {
- return graphNodes;
- }
- };
+ final MigrationDataProvider<GraphNode> testMigrationProvider = () -> graphNodes;
final TestProgressObserver progressObserver = new TestProgressObserver();
-
//read everything in previous version format and put it into our types.
- final int returned = edgeDataMigrationImpl.migrate( GraphDataVersions.INITIAL.getVersion() , testMigrationProvider, progressObserver );
+ EdgeDataMigrationImpl edgeDataMigrationImpl =
+ new EdgeDataMigrationImpl( keyspace, graphManagerFactory, edgesFromSourceObservable, allVersions,
+ edgeMetadataSerializationV2, testMigrationProvider );
+
+
+ final int returned = edgeDataMigrationImpl.migrate( GraphDataVersions.INITIAL.getVersion(), progressObserver );
//perform the migration
assertEquals( "Correct version returned", returned, GraphDataVersions.META_SHARDING.getVersion() );
@@ -164,9 +176,6 @@ public class EdgeDataMigrationImplTest implements DataMigrationResetRule.DataMig
assertTrue( "Edge type present", source2Edges.contains( "foo" ) );
assertTrue( "Edge type present", source2Edges.contains( "bar" ) );
-
-
-
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/d971ec50/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
index d03bbf8..7279174 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
@@ -56,8 +56,8 @@ public abstract class IndexModule extends AbstractModule {
//wire up the edg migration. A no-op ATM, but retained for future development
- Multibinder<DataMigration<ApplicationScope>> dataMigrationMultibinder =
- Multibinder.newSetBinder( binder(), new TypeLiteral<DataMigration<ApplicationScope>>() {}, IndexMigration.class );
+ Multibinder<DataMigration> dataMigrationMultibinder =
+ Multibinder.newSetBinder( binder(), new TypeLiteral<DataMigration>() {}, IndexMigration.class );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/d971ec50/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexMigrationPlugin.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexMigrationPlugin.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexMigrationPlugin.java
index a28c701..c8e353d 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexMigrationPlugin.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexMigrationPlugin.java
@@ -18,20 +18,18 @@ package org.apache.usergrid.persistence.index.migration;
import com.google.inject.Inject;
import org.apache.usergrid.persistence.core.migration.data.*;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import java.util.Set;
/**
* Classy class class.
*/
-public class EsIndexMigrationPlugin extends AbstractMigrationPlugin<ApplicationScope>{
+public class EsIndexMigrationPlugin extends AbstractMigrationPlugin {
@Inject
- public EsIndexMigrationPlugin(@IndexMigration final Set<DataMigration<ApplicationScope>> entityDataMigrations,
- final MigrationDataProvider<ApplicationScope> entityIdScopeDataMigrationProvider,
+ public EsIndexMigrationPlugin(@IndexMigration final Set<DataMigration> entityDataMigrations,
final MigrationInfoSerialization migrationInfoSerialization ){
- super(entityDataMigrations,entityIdScopeDataMigrationProvider,migrationInfoSerialization);
+ super(entityDataMigrations,migrationInfoSerialization);
}
@Override
[06/13] usergrid git commit: Merge branch 'two-dot-o-dev' of
https://git-wip-us.apache.org/repos/asf/usergrid into USERGRID-1018
Posted by to...@apache.org.
Merge branch 'two-dot-o-dev' of https://git-wip-us.apache.org/repos/asf/usergrid into USERGRID-1018
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/fb1cf2e9
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/fb1cf2e9
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/fb1cf2e9
Branch: refs/heads/two-dot-o-dev
Commit: fb1cf2e939fc123b9468df372c80664459ef0b96
Parents: b7a22c2 723ee3a
Author: Todd Nine <tn...@apigee.com>
Authored: Tue Sep 15 17:27:20 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Tue Sep 15 17:27:20 2015 -0600
----------------------------------------------------------------------
.../rx/impl/AllApplicationsObservableImpl.java | 36 +++-----
.../corepersistence/CpEntityMapUtilsTest.java | 4 +-
.../org/apache/usergrid/persistence/GeoIT.java | 52 ++++++++++-
.../migration/MvccEntityDataMigrationImpl.java | 94 +++++++++++---------
.../model/entity/MapToEntityConverter.java | 8 +-
5 files changed, 119 insertions(+), 75 deletions(-)
----------------------------------------------------------------------