You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by to...@apache.org on 2014/10/30 04:40:11 UTC
[45/45] git commit: Added migration to system resource and emf
Added migration to system resource and emf
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/5c56fa01
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/5c56fa01
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/5c56fa01
Branch: refs/heads/key-row-sharding
Commit: 5c56fa016a231fb3884c7fcfa93371c5dfa47d1b
Parents: f85fed1
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Oct 29 21:39:12 2014 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Oct 29 21:39:12 2014 -0600
----------------------------------------------------------------------
.../corepersistence/CpEntityManagerFactory.java | 22 +++++
.../usergrid/corepersistence/GuiceModule.java | 8 ++
.../HybridEntityManagerFactory.java | 19 ++++
.../migration/GraphShardVersionMigration.java | 17 +++-
.../corepersistence/migration/Versions.java | 35 +++++++-
.../persistence/EntityManagerFactory.java | 18 ++++
.../cassandra/EntityManagerFactoryImpl.java | 19 ++++
.../data/DataMigrationManagerImpl.java | 2 +
.../apache/usergrid/rest/SystemResource.java | 94 +++++++++++++++-----
9 files changed, 212 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5c56fa01/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index 6e6b461..86e7c31 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -52,6 +52,7 @@ import org.apache.usergrid.persistence.cassandra.Setup;
import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl;
+import org.apache.usergrid.persistence.core.migration.data.DataMigrationManager;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
import org.apache.usergrid.persistence.entities.Application;
@@ -122,6 +123,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
private CpManagerCache managerCache;
+ private DataMigrationManager dataMigrationManager;
CassandraService cass;
CounterUtils counterUtils;
@@ -183,6 +185,8 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
mmf = injector.getInstance( MapManagerFactory.class );
managerCache = new CpManagerCache( ecmf, eif, gmf, mmf );
+
+ dataMigrationManager = injector.getInstance( DataMigrationManager.class );
}
return managerCache;
}
@@ -707,6 +711,24 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
@Override
+ public void migrateData() throws Exception {
+ dataMigrationManager.migrate();
+ }
+
+
+ @Override
+ public String getMigrateDataStatus() {
+ return dataMigrationManager.getLastStatus();
+ }
+
+
+ @Override
+ public int getMigrateDataVersion() {
+ return dataMigrationManager.getCurrentVersion();
+ }
+
+
+ @Override
public void flushEntityManagerCaches() {
Map<UUID, EntityManager> entityManagersMap = entityManagers.asMap();
for ( UUID appUuid : entityManagersMap.keySet() ) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5c56fa01/stack/core/src/main/java/org/apache/usergrid/corepersistence/GuiceModule.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/GuiceModule.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/GuiceModule.java
index 2221f80..fb4800c 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/GuiceModule.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/GuiceModule.java
@@ -18,8 +18,12 @@ package org.apache.usergrid.corepersistence;
import com.google.inject.AbstractModule;
+import com.google.inject.multibindings.Multibinder;
+
+import org.apache.usergrid.corepersistence.migration.GraphShardVersionMigration;
import org.apache.usergrid.persistence.collection.guice.CollectionModule;
import org.apache.usergrid.persistence.core.guice.CommonModule;
+import org.apache.usergrid.persistence.core.migration.data.DataMigration;
import org.apache.usergrid.persistence.graph.guice.GraphModule;
import org.apache.usergrid.persistence.index.guice.IndexModule;
import org.apache.usergrid.persistence.map.guice.MapModule;
@@ -47,6 +51,10 @@ public class GuiceModule extends AbstractModule {
bind(CpEntityDeleteListener.class).asEagerSingleton();
bind(CpEntityIndexDeleteListener.class).asEagerSingleton();
+ Multibinder<DataMigration> dataMigrationMultibinder = Multibinder.newSetBinder( binder(), DataMigration.class );
+ dataMigrationMultibinder.addBinding().to( GraphShardVersionMigration.class );
+
+
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5c56fa01/stack/core/src/main/java/org/apache/usergrid/corepersistence/HybridEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/HybridEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/HybridEntityManagerFactory.java
index 54a5dee..ebb0ee6 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/HybridEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/HybridEntityManagerFactory.java
@@ -159,6 +159,25 @@ public class HybridEntityManagerFactory implements EntityManagerFactory, Applica
factory.rebuildApplicationIndexes(appId, po);
}
+
+ @Override
+ public void migrateData() throws Exception {
+ factory.migrateData();
+ }
+
+
+ @Override
+ public String getMigrateDataStatus() {
+ return factory.getMigrateDataStatus();
+ }
+
+
+ @Override
+ public int getMigrateDataVersion() {
+ return factory.getMigrateDataVersion();
+ }
+
+
@Override
public void rebuildCollectionIndex(UUID appId, String collection, ProgressObserver po) {
factory.rebuildCollectionIndex(appId, collection, po);
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5c56fa01/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/GraphShardVersionMigration.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/GraphShardVersionMigration.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/GraphShardVersionMigration.java
index e714b5e..92982ba 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/GraphShardVersionMigration.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/GraphShardVersionMigration.java
@@ -22,7 +22,22 @@
package org.apache.usergrid.corepersistence.migration;
+import org.apache.usergrid.persistence.core.migration.data.DataMigration;
+
+
/**
* Migration for migrating graph edges to the new Shards
*/
-public class GraphShardVersionMigration {}
+public class GraphShardVersionMigration implements DataMigration{
+
+ @Override
+ public void migrate( final ProgressObserver observer ) throws Throwable {
+
+ }
+
+
+ @Override
+ public int getVersion() {
+ return Versions.VERSION_1;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5c56fa01/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/Versions.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/Versions.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/Versions.java
index 56bd65e..e9ee517 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/Versions.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/Versions.java
@@ -1,4 +1,37 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one
+ * * or more contributor license agreements. See the NOTICE file
+ * * distributed with this work for additional information
+ * * regarding copyright ownership. The ASF licenses this file
+ * * to you under the Apache License, Version 2.0 (the
+ * * "License"); you may not use this file except in compliance
+ * * with the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing,
+ * * software distributed under the License is distributed on an
+ * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * * KIND, either express or implied. See the License for the
+ * * specific language governing permissions and limitations
+ * * under the License.
+ *
+ */
+
package org.apache.usergrid.corepersistence.migration;
-public class Versions {}
+import org.apache.usergrid.persistence.graph.serialization.impl.EdgeMetadataSerializationProxyImpl;
+
+
+/**
+ * Simple class to hold the constants of all versions
+ */
+public class Versions {
+
+ /**
+ * Version 1
+ */
+ public static final int VERSION_1 = EdgeMetadataSerializationProxyImpl.MIGRATION_VERSION;
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5c56fa01/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
index e57aa69..e7c0874 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
@@ -123,6 +123,24 @@ public interface EntityManagerFactory {
public void rebuildApplicationIndexes( UUID appId, ProgressObserver po ) throws Exception;
+ /**
+ * Perform any data migrations necessary in the system
+ * @throws Exception
+ */
+ public void migrateData() throws Exception;
+
+ /**
+ * Return the migration status message
+ * @return
+ */
+ public String getMigrateDataStatus();
+
+ /**
+ * Return the current migration version of the system
+ * @return
+ */
+ public int getMigrateDataVersion();
+
public void setApplicationContext(ApplicationContext ac);
/** For testing purposes */
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5c56fa01/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
index 399bccd..f09c1eb 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
@@ -434,6 +434,25 @@ public class EntityManagerFactoryImpl implements EntityManagerFactory, Applicati
throw new UnsupportedOperationException("Not supported.");
}
+
+ @Override
+ public void migrateData() throws Exception {
+
+ }
+
+
+ @Override
+ public String getMigrateDataStatus() {
+ return null;
+ }
+
+
+ @Override
+ public int getMigrateDataVersion() {
+ return 0;
+ }
+
+
@Override
public void rebuildCollectionIndex(UUID appId, String collection, ProgressObserver po) {
throw new UnsupportedOperationException("Not supported.");
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5c56fa01/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManagerImpl.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManagerImpl.java
index e739648..cf0d15d 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManagerImpl.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManagerImpl.java
@@ -151,6 +151,8 @@ public class DataMigrationManagerImpl implements DataMigrationManager {
//set the version
migrationInfoSerialization.setVersion( migrationVersion );
+ versionCache.invalidateAll();
+
//update the observer for progress so other nodes can see it
observer.update( migrationVersion, "Completed successfully" );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5c56fa01/stack/rest/src/main/java/org/apache/usergrid/rest/SystemResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/SystemResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/SystemResource.java
index d068846..bdf647b 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/SystemResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/SystemResource.java
@@ -38,12 +38,14 @@ import org.springframework.stereotype.Component;
import org.apache.usergrid.persistence.EntityManager;
import org.apache.usergrid.persistence.EntityManagerFactory;
+import org.apache.usergrid.persistence.EntityManagerFactory.ProgressObserver;
import org.apache.usergrid.persistence.EntityRef;
import org.apache.usergrid.persistence.index.utils.UUIDUtils;
import org.apache.usergrid.rest.security.annotations.RequireSystemAccess;
+import com.fasterxml.jackson.databind.node.JsonNodeFactory;
+import com.fasterxml.jackson.databind.node.ObjectNode;
import com.sun.jersey.api.json.JSONWithPadding;
-import org.apache.usergrid.persistence.EntityManagerFactory.ProgressObserver;
@Path( "/system" )
@@ -67,7 +69,7 @@ public class SystemResource extends AbstractContextResource {
@GET
@Path( "database/setup" )
public JSONWithPadding getSetup( @Context UriInfo ui,
- @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
throws Exception {
ApiResponse response = createApiResponse();
@@ -91,7 +93,7 @@ public class SystemResource extends AbstractContextResource {
@GET
@Path( "superuser/setup" )
public JSONWithPadding getSetupSuperuser( @Context UriInfo ui,
- @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
throws Exception {
ApiResponse response = createApiResponse();
@@ -114,9 +116,67 @@ public class SystemResource extends AbstractContextResource {
@RequireSystemAccess
@PUT
+ @Path( "migrate/run" )
+ public JSONWithPadding migrateData( @Context UriInfo ui,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+ throws Exception {
+
+ ApiResponse response = createApiResponse();
+ response.setAction( "Migrate Data" );
+
+
+ final Thread migrate = new Thread() {
+
+ @Override
+ public void run() {
+ logger.info( "Rebuilding all indexes" );
+
+ try {
+ emf.migrateData();
+ }
+ catch ( Exception e ) {
+ logger.error( "Unable to rebuild indexes", e );
+ }
+ }
+ };
+
+ migrate.setName( "Index migrate data formats" );
+ migrate.setDaemon( true );
+ migrate.start();
+
+
+ response.setSuccess();
+
+ return new JSONWithPadding( response, callback );
+ }
+
+
+ @RequireSystemAccess
+ @GET
+ @Path( "migrate/status" )
+ public JSONWithPadding migrateStatus( @Context UriInfo ui,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+ throws Exception {
+
+ ApiResponse response = createApiResponse();
+ response.setAction( "Migrate Schema indexes" );
+
+ ObjectNode node = JsonNodeFactory.instance.objectNode();
+ node.put( "currentVersion", emf.getMigrateDataVersion() );
+ node.put( "lastMessage", emf.getMigrateDataStatus() );
+ response.setProperty( "status", node );
+
+ response.setSuccess();
+
+ return new JSONWithPadding( response, callback );
+ }
+
+
+ @RequireSystemAccess
+ @PUT
@Path( "index/rebuild" )
public JSONWithPadding rebuildIndexes( @Context UriInfo ui,
- @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
throws Exception {
ApiResponse response = createApiResponse();
@@ -168,11 +228,9 @@ public class SystemResource extends AbstractContextResource {
@RequireSystemAccess
@PUT
@Path( "index/rebuild/" + RootResource.APPLICATION_ID_PATH )
- public JSONWithPadding rebuildIndexes(
- @Context UriInfo ui,
- @PathParam( "applicationId" ) String applicationIdStr,
- @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback,
- @QueryParam( "delay" ) @DefaultValue( "10" ) final long delay)
+ public JSONWithPadding rebuildIndexes( @Context UriInfo ui, @PathParam( "applicationId" ) String applicationIdStr,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback,
+ @QueryParam( "delay" ) @DefaultValue( "10" ) final long delay )
throws Exception {
@@ -181,8 +239,6 @@ public class SystemResource extends AbstractContextResource {
response.setAction( "rebuild indexes" );
-
-
final EntityManager em = emf.getEntityManager( appId );
final Set<String> collectionNames = em.getApplicationCollections();
@@ -213,12 +269,11 @@ public class SystemResource extends AbstractContextResource {
@RequireSystemAccess
@PUT
@Path( "index/rebuild/" + RootResource.APPLICATION_ID_PATH + "/{collectionName}" )
- public JSONWithPadding rebuildIndexes(
- @Context UriInfo ui,
- @PathParam( "applicationId" ) final String applicationIdStr,
- @PathParam( "collectionName" ) final String collectionName,
- @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback,
- @QueryParam( "delay" ) @DefaultValue( "10" ) final long delay )
+ public JSONWithPadding rebuildIndexes( @Context UriInfo ui,
+ @PathParam( "applicationId" ) final String applicationIdStr,
+ @PathParam( "collectionName" ) final String collectionName,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback,
+ @QueryParam( "delay" ) @DefaultValue( "10" ) final long delay )
throws Exception {
final UUID appId = UUIDUtils.tryExtractUUID( applicationIdStr );
@@ -233,8 +288,7 @@ public class SystemResource extends AbstractContextResource {
}
};
- rebuild.setName( String.format(
- "Index rebuild for app %s and collection %s", appId, collectionName ) );
+ rebuild.setName( String.format( "Index rebuild for app %s and collection %s", appId, collectionName ) );
rebuild.setDaemon( true );
rebuild.start();
@@ -249,7 +303,7 @@ public class SystemResource extends AbstractContextResource {
@Override
public void onProgress( final EntityRef entity ) {
- logger.info( "Indexing entity {}:{}", entity.getType(), entity.getUuid());
+ logger.info( "Indexing entity {}:{}", entity.getType(), entity.getUuid() );
}