You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by to...@apache.org on 2015/06/09 21:34:38 UTC
incubator-usergrid git commit: Fixes incorrect collection of app name
/uuids
Repository: incubator-usergrid
Updated Branches:
refs/heads/USERGRID-688 9337b37f6 -> 054405e42
Fixes incorrect collection of app name /uuids
Fixes incorrect usage of timestamps when creating collections and connections so that mark and sweep works as expected.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/054405e4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/054405e4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/054405e4
Branch: refs/heads/USERGRID-688
Commit: 054405e42ee14ec61a8f1237b187c78c8a451edf
Parents: 9337b37
Author: Todd Nine <tn...@apigee.com>
Authored: Tue Jun 9 13:34:37 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Tue Jun 9 13:34:37 2015 -0600
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 6 +-
.../corepersistence/CpEntityManagerFactory.java | 161 ++++++++++---------
.../corepersistence/util/CpNamingUtils.java | 13 +-
.../persistence/EntityManagerFactory.java | 18 +--
.../cassandra/EntityManagerFactoryImplIT.java | 1 +
5 files changed, 103 insertions(+), 96 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/054405e4/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 7cee908..022b14d 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -129,6 +129,7 @@ import static org.apache.commons.lang.StringUtils.capitalize;
import static org.apache.commons.lang.StringUtils.isBlank;
import static org.apache.usergrid.corepersistence.util.CpEntityMapUtils.entityToCpEntity;
import static org.apache.usergrid.corepersistence.util.CpNamingUtils.createConnectionTypeSearch;
+import static org.apache.usergrid.corepersistence.util.CpNamingUtils.createGraphOperationTimestamp;
import static org.apache.usergrid.corepersistence.util.CpNamingUtils.getConnectionNameFromEdgeName;
import static org.apache.usergrid.persistence.Schema.COLLECTION_ROLES;
import static org.apache.usergrid.persistence.Schema.COLLECTION_USERS;
@@ -689,11 +690,8 @@ public class CpEntityManager implements EntityManager {
Id entityId = new SimpleId( entityRef.getUuid(), entityRef.getType() );
- //for devices we don't have time uuids
- UUID timeUUID = UUIDUtils.newTimeUUID();
-
//Step 1 & 2 of delete
- return ecm.mark( entityId ).mergeWith( gm.markNode( entityId, timeUUID.timestamp() ) );
+ return ecm.mark( entityId ).mergeWith( gm.markNode( entityId, createGraphOperationTimestamp() ) );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/054405e4/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index 762a5cd..ef1ffc0 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -23,12 +23,8 @@ import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import java.util.UUID;
-import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
-import org.apache.usergrid.corepersistence.index.ReIndexRequestBuilder;
-import org.apache.usergrid.persistence.*;
-import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdge;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.BeansException;
@@ -38,10 +34,20 @@ import org.springframework.context.ApplicationContextAware;
import org.apache.commons.lang.StringUtils;
import org.apache.usergrid.corepersistence.asyncevents.AsyncEventService;
+import org.apache.usergrid.corepersistence.index.ReIndexRequestBuilder;
import org.apache.usergrid.corepersistence.index.ReIndexService;
import org.apache.usergrid.corepersistence.pipeline.builder.PipelineBuilderFactory;
import org.apache.usergrid.corepersistence.util.CpNamingUtils;
import org.apache.usergrid.exception.ConflictException;
+import org.apache.usergrid.persistence.AbstractEntity;
+import org.apache.usergrid.persistence.Entity;
+import org.apache.usergrid.persistence.EntityFactory;
+import org.apache.usergrid.persistence.EntityManager;
+import org.apache.usergrid.persistence.EntityManagerFactory;
+import org.apache.usergrid.persistence.EntityRef;
+import org.apache.usergrid.persistence.Query;
+import org.apache.usergrid.persistence.Results;
+import org.apache.usergrid.persistence.SimpleEntityRef;
import org.apache.usergrid.persistence.cassandra.CassandraService;
import org.apache.usergrid.persistence.cassandra.CounterUtils;
import org.apache.usergrid.persistence.cassandra.Setup;
@@ -308,8 +314,8 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
// find application_info for application to delete
- migrateAppInfo(applicationId, CpNamingUtils.APPLICATION_INFO, CpNamingUtils.DELETED_APPLICATION_INFO).toBlocking()
- .lastOrDefault(null);
+ migrateAppInfo(applicationId, CpNamingUtils.APPLICATION_INFO, CpNamingUtils.DELETED_APPLICATION_INFOS, CpNamingUtils.DELETED_APPLICATION_INFO).toBlocking()
+ .lastOrDefault( null );
}
//TODO: return status for restore
@@ -317,48 +323,67 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
public Entity restoreApplication(UUID applicationId) throws Exception {
// get the deleted_application_info for the deleted app
- return (Entity) migrateAppInfo(applicationId, CpNamingUtils.DELETED_APPLICATION_INFO, CpNamingUtils.APPLICATION_INFO)
- .map(o -> {
- final ReIndexRequestBuilder builder =
- reIndexService.getBuilder().withApplicationId(applicationId);
- return reIndexService.rebuildIndex(builder);
- })
- .map(status -> {
- final EntityManager managementEm = getEntityManager(getManagementAppId());
- try {
- return managementEm.get(new SimpleEntityRef(CpNamingUtils.APPLICATION_INFO, applicationId));
- } catch (Exception e) {
- logger.error("Failed to get entity", e);
- throw new RuntimeException(e);
- }
- })
+ return (Entity) migrateAppInfo( applicationId, CpNamingUtils.DELETED_APPLICATION_INFO,
+ CpNamingUtils.APPLICATION_INFOS , CpNamingUtils.APPLICATION_INFO ).lastOrDefault( null )
+ .map( appInfo -> {
+
+ //start the index rebuild
+ final ReIndexRequestBuilder builder = reIndexService.getBuilder().withApplicationId( applicationId );
+ reIndexService.rebuildIndex( builder );
+
+ //load the entity
+ final EntityManager managementEm = getEntityManager( getManagementAppId() );
+ try {
+ return managementEm.get( new SimpleEntityRef( CpNamingUtils.APPLICATION_INFO, applicationId ) );
+ }
+ catch ( Exception e ) {
+ logger.error( "Failed to get entity", e );
+ throw new RuntimeException( e );
+ }
+ } )
.toBlocking().lastOrDefault(null);
}
- @Override
- public Observable migrateAppInfo(UUID applicationUUID, String collectionFromName, String collectionToName) throws Exception {
+// @Override
+
+
+ /**
+ * Migrate the application from one type to another. Used in delete and restore
+ * @param applicationUUID The applicationUUID
+ * @param deleteTypeName The type to use on the delete
+ * @param createCollectionName The name of the collection to write the entity into
+ * @param createTypeName The type to use on the create
+ * @return
+ * @throws Exception
+ */
+ private Observable migrateAppInfo(final UUID applicationUUID, final String deleteTypeName, final String createCollectionName, final String createTypeName ) throws Exception {
final ApplicationScope managementAppScope = CpNamingUtils.getApplicationScope(CpNamingUtils.MANAGEMENT_APPLICATION_ID);
final EntityManager managementEm = getEntityManager(CpNamingUtils.MANAGEMENT_APPLICATION_ID);
- final Id applicationId = new SimpleId(applicationUUID, collectionFromName);
- final ApplicationScope applicationScope = new ApplicationScopeImpl(applicationId);
+ //the application id we will be removing
+ final Id deleteApplicationId = new SimpleId(applicationUUID, deleteTypeName );
+
+ //the application id we'll be creating
+ final Id createApplicationId = new SimpleId( applicationUUID, createTypeName );
- Entity oldAppEntity = managementEm.get(new SimpleEntityRef(collectionFromName, applicationUUID));
+ //the application scope of the deleted app to clean it's index
+ final ApplicationScope deleteApplicationScope = new ApplicationScopeImpl(deleteApplicationId);
+
+ Entity oldAppEntity = managementEm.get(new SimpleEntityRef( deleteTypeName, applicationUUID));
Observable copyConnections = Observable.empty();
if (oldAppEntity != null) {
// ensure that there is not already a deleted app with the same name
- final EntityRef alias = managementEm.getAlias(collectionToName, oldAppEntity.getName());
+ final EntityRef alias = managementEm.getAlias( createCollectionName, oldAppEntity.getName());
if (alias != null) {
throw new ConflictException("Cannot delete app with same name as already deleted app");
}
// make a copy of the app to delete application_info entity
// and put it in a deleted_application_info collection
- final Entity newAppEntity = managementEm.create(new SimpleId(applicationUUID,
- collectionToName), oldAppEntity.getProperties());
+ final Entity newAppEntity = managementEm.create(new SimpleId(applicationUUID, createTypeName ), oldAppEntity.getProperties());
// copy its connections too
@@ -381,15 +406,15 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
}
final Id managementAppId = CpNamingUtils.getManagementApplicationId();
- final ApplicationEntityIndex aei = entityIndexFactory.createApplicationEntityIndex(applicationScope);
+ final ApplicationEntityIndex aei = entityIndexFactory.createApplicationEntityIndex(deleteApplicationScope);
final GraphManager managementGraphManager = managerCache.getGraphManager(managementAppScope);
- final Edge createEdge = CpNamingUtils.createCollectionEdge(managementAppId, collectionToName, applicationId);
+ final Edge createEdge = CpNamingUtils.createCollectionEdge(managementAppId, createCollectionName, createApplicationId);
//TODO T.N. Removing this causes a failure
// final Observable compactObservable = managementGraphManager.compactNode(applicationId);
final Observable deleteNodeGraph = managementGraphManager
- .markNode( applicationId, CpNamingUtils.createGraphOperationTimestamp() );
+ .markNode( deleteApplicationId, CpNamingUtils.createGraphOperationTimestamp() );
// .flatMap(id -> compactObservable);
final Observable createNodeGraph = managementGraphManager.writeEdge(createEdge);
@@ -397,18 +422,19 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
final Observable deleteAppFromIndex = aei.deleteApplication();
return Observable
- .merge(copyConnections, createNodeGraph, deleteNodeGraph, deleteAppFromIndex)
- .doOnCompleted(() -> {
+ .merge( copyConnections, createNodeGraph, deleteNodeGraph, deleteAppFromIndex )
+ .doOnCompleted( () -> {
try {
- if (oldAppEntity != null) {
- managementEm.delete(oldAppEntity);
- applicationIdCache.evictAppId(oldAppEntity.getName());
+ if ( oldAppEntity != null ) {
+ managementEm.delete( oldAppEntity );
+ applicationIdCache.evictAppId( oldAppEntity.getName() );
}
entityIndex.refreshAsync().toBlocking().last();
- } catch (Exception e) {
- throw new RuntimeException(e);
}
- });
+ catch ( Exception e ) {
+ throw new RuntimeException( e );
+ }
+ } );
}
@@ -428,19 +454,17 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
@Override
public Map<String, UUID> getApplications() throws Exception {
- return getApplications(false);
+ return getApplications(CpNamingUtils.getEdgeTypeFromCollectionName( CpNamingUtils.APPLICATION_INFOS ));
}
@Override
public Map<String, UUID> getDeletedApplications() throws Exception {
- return getApplications( true );
+ return getApplications( CpNamingUtils.getEdgeTypeFromCollectionName( CpNamingUtils.DELETED_APPLICATION_INFOS ) );
}
- public Map<String, UUID> getApplications(boolean deleted) throws Exception {
-
- Map<String, UUID> appMap = new HashMap<>();
+ private Map<String, UUID> getApplications(final String edgeType) throws Exception {
ApplicationScope appScope =
CpNamingUtils.getApplicationScope(CpNamingUtils.MANAGEMENT_APPLICATION_ID);
@@ -454,13 +478,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
}
Id managementId = new SimpleId( managementApp.getUuid(), managementApp.getType() );
- final String edgeType;
- if ( deleted ) {
- edgeType = CpNamingUtils.getEdgeTypeFromCollectionName( CpNamingUtils.DELETED_APPLICATION_INFOS );
- } else {
- edgeType = CpNamingUtils.getEdgeTypeFromCollectionName( CpNamingUtils.APPLICATION_INFOS );
- }
logger.debug("getApplications(): Loading edges of edgeType {} from {}:{}",
new Object[]{edgeType, managementId.getType(), managementId.getUuid()});
@@ -469,33 +487,30 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
managementId, edgeType, Long.MAX_VALUE,
SearchByEdgeType.Order.DESCENDING, Optional.<Edge>absent() ));
- // TODO This is wrong, and will result in OOM if there are too many applications.
- // This needs to stream properly with a buffer
-
- edges.doOnNext(edge -> {
- Id targetId = edge.getTargetNode();
+ final EntityCollectionManager ecm = managerCache.getEntityCollectionManager( appScope );
- logger.debug("getApplications(): Processing edge from {}:{} to {}:{}", new Object[]{
- edge.getSourceNode().getType(), edge.getSourceNode().getUuid(),
- edge.getTargetNode().getType(), edge.getTargetNode().getUuid()
- });
-
- org.apache.usergrid.persistence.model.entity.Entity appInfo =
- managerCache.getEntityCollectionManager(appScope).load(targetId)
- .toBlocking().lastOrDefault(null);
+ //buffer our edges and batch fetch the app infos for faster I/O
+ return edges.map( edge -> {
+ return edge.getTargetNode();
+ } ).buffer( 100 ).flatMap( entityIds -> {
+ return ecm.load( entityIds );
+ } )
+ .flatMap( entitySet -> Observable.from( entitySet.getEntities() ) )
+ //collect all the app infos into a single map for return
+ .collect( () -> new HashMap<String, UUID>(), ( appMap, entity ) -> {
- if (appInfo == null) {
- logger.warn("Application {} has edge but not found in em", targetId);
- return;
- }
+ if ( !entity.getEntity().isPresent() ) {
+ return;
+ }
- UUID applicationId = UUIDUtils.tryExtractUUID(
- appInfo.getField(PROPERTY_APPLICATION_ID).getValue().toString());
+ final org.apache.usergrid.persistence.model.entity.Entity entityData =
+ entity.getEntity().get();
- appMap.put((String) appInfo.getField(PROPERTY_NAME).getValue(), applicationId);
- }).toBlocking().lastOrDefault(null);
+ final UUID applicationId = ( UUID ) entityData.getField( PROPERTY_APPLICATION_ID ).getValue();
+ final String applicationName = ( String ) entityData.getField( PROPERTY_NAME ).getValue();
- return appMap;
+ appMap.put( applicationName , applicationId );
+ } ).toBlocking().last();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/054405e4/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
index f1b1700..24fdbab 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
@@ -169,15 +169,8 @@ public class CpNamingUtils {
final String edgeType = CpNamingUtils.getEdgeTypeFromCollectionName( collectionName );
- //if they don't use a time based uuid (such as in devices) we need to create a timestamp from "now" since
- // this is when the entity
- //will be added to the collection
- final UUID timeStampUuid = UUIDGenerator.newTimeUUID();
-
- final long uuidTimestamp = UUIDUtils.getUUIDLong( timeStampUuid );
-
// create graph edge connection from head entity to member entity
- return new SimpleEdge( sourceId, edgeType, entityId, uuidTimestamp );
+ return new SimpleEdge( sourceId, edgeType, entityId, createGraphOperationTimestamp() );
}
@@ -198,7 +191,7 @@ public class CpNamingUtils {
final String edgeType = getEdgeTypeFromConnectionType( connectionType );
// create graph edge connection from head entity to member entity
- return new SimpleEdge( sourceEntityId, edgeType, targetEntityId, UUIDGenerator.newTimeUUID().timestamp() );
+ return new SimpleEdge( sourceEntityId, edgeType, targetEntityId, createGraphOperationTimestamp() );
}
@@ -207,7 +200,7 @@ public class CpNamingUtils {
* @return
*/
public static long createGraphOperationTimestamp(){
- return UUIDUtils.newTimeUUID().timestamp();
+ return UUIDGenerator.newTimeUUID().timestamp();
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/054405e4/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
index cd7e515..b09b355 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
@@ -92,15 +92,15 @@ public interface EntityManagerFactory {
*/
void deleteApplication( UUID applicationId ) throws Exception;
- /**
- *
- * @param applicationUUID
- * @param collectionFromName
- * @param collectionToName
- * @return
- * @throws Exception
- */
- Observable migrateAppInfo( UUID applicationUUID, String collectionFromName, String collectionToName) throws Exception;
+// /**
+// *
+// * @param applicationUUID
+// * @param collectionFromName
+// * @param collectionToName
+// * @return
+// * @throws Exception
+// */
+// Observable migrateAppInfo( UUID applicationUUID, String collectionFromName, String collectionToName) throws Exception;
/**
* Restore deleted application.
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/054405e4/stack/core/src/test/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImplIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImplIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImplIT.java
index 5b09eac..83760e2 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImplIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImplIT.java
@@ -209,6 +209,7 @@ public class EntityManagerFactoryImplIT extends AbstractCoreIT {
apps = setup.getEmf().getApplications();
found = findApps.call(deletedAppId, apps);
+
assertTrue("Restored app not found in apps collection", found);
// TODO: this assertion should work!