You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sn...@apache.org on 2014/10/02 06:12:13 UTC
[04/12] git commit: Changes to 1) ensure that the CpEntityManager's
ElasticSearch can be rebuilt entirely from data stored in Cassandra,
2) provide support Index Rebuild in two-dot-o and the beginnings of an Index
Rebuild test.
Changes to 1) ensure that the CpEntityManager's ElasticSearch can be rebuilt entirely from data stored in Cassandra, 2) provide support Index Rebuild in two-dot-o and the beginnings of an Index Rebuild test.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/1aa04a71
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/1aa04a71
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/1aa04a71
Branch: refs/heads/two-dot-o
Commit: 1aa04a71d73b30f647b6d2d2ea8ce068557da31a
Parents: 58fc540
Author: Dave Johnson <dm...@apigee.com>
Authored: Wed Oct 1 10:47:52 2014 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Wed Oct 1 10:47:52 2014 -0400
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 479 +++++++++++---
.../corepersistence/CpEntityManagerFactory.java | 639 ++++++-------------
.../corepersistence/CpRelationManager.java | 356 ++++-------
.../usergrid/corepersistence/CpSetup.java | 55 +-
.../HybridEntityManagerFactory.java | 15 +-
.../persistence/EntityManagerFactory.java | 9 +-
.../persistence/cassandra/CassandraService.java | 5 -
.../cassandra/EntityManagerFactoryImpl.java | 54 +-
.../PerformanceEntityRebuildIndexTest.java | 64 +-
.../org/apache/usergrid/tools/IndexRebuild.java | 62 +-
10 files changed, 835 insertions(+), 903 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1aa04a71/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 51e660b..1c54d5f 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -16,9 +16,13 @@
package org.apache.usergrid.corepersistence;
+import com.netflix.hystrix.exception.HystrixRuntimeException;
+import com.yammer.metrics.annotation.Metered;
+import static java.lang.String.CASE_INSENSITIVE_ORDER;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Arrays;
+import static java.util.Arrays.asList;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
@@ -32,11 +36,26 @@ import java.util.Set;
import java.util.TreeMap;
import java.util.TreeSet;
import java.util.UUID;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.util.Assert;
-
+import me.prettyprint.hector.api.Keyspace;
+import me.prettyprint.hector.api.beans.ColumnSlice;
+import me.prettyprint.hector.api.beans.CounterRow;
+import me.prettyprint.hector.api.beans.CounterRows;
+import me.prettyprint.hector.api.beans.CounterSlice;
+import me.prettyprint.hector.api.beans.DynamicComposite;
+import me.prettyprint.hector.api.beans.HColumn;
+import me.prettyprint.hector.api.beans.HCounterColumn;
+import me.prettyprint.hector.api.factory.HFactory;
+import static me.prettyprint.hector.api.factory.HFactory.createCounterSliceQuery;
+import static me.prettyprint.hector.api.factory.HFactory.createMutator;
+import me.prettyprint.hector.api.mutation.Mutator;
+import me.prettyprint.hector.api.query.MultigetSliceCounterQuery;
+import me.prettyprint.hector.api.query.QueryResult;
+import me.prettyprint.hector.api.query.SliceCounterQuery;
+import static org.apache.commons.lang.StringUtils.capitalize;
+import static org.apache.commons.lang.StringUtils.isBlank;
+import static org.apache.usergrid.corepersistence.CpRelationManager.ALL_TYPES;
+import static org.apache.usergrid.corepersistence.CpRelationManager.EDGE_COLL_SUFFIX;
+import static org.apache.usergrid.corepersistence.CpRelationManager.EDGE_CONN_SUFFIX;
import org.apache.usergrid.persistence.AggregateCounter;
import org.apache.usergrid.persistence.AggregateCounterSet;
import org.apache.usergrid.persistence.CollectionRef;
@@ -50,14 +69,43 @@ import org.apache.usergrid.persistence.EntityRef;
import org.apache.usergrid.persistence.IndexBucketLocator;
import org.apache.usergrid.persistence.RelationManager;
import org.apache.usergrid.persistence.Results;
+import org.apache.usergrid.persistence.Schema;
+import static org.apache.usergrid.persistence.Schema.COLLECTION_ROLES;
+import static org.apache.usergrid.persistence.Schema.COLLECTION_USERS;
+import static org.apache.usergrid.persistence.Schema.DICTIONARY_PERMISSIONS;
+import static org.apache.usergrid.persistence.Schema.DICTIONARY_ROLENAMES;
+import static org.apache.usergrid.persistence.Schema.DICTIONARY_ROLETIMES;
+import static org.apache.usergrid.persistence.Schema.DICTIONARY_SETS;
+import static org.apache.usergrid.persistence.Schema.PROPERTY_CREATED;
+import static org.apache.usergrid.persistence.Schema.PROPERTY_INACTIVITY;
+import static org.apache.usergrid.persistence.Schema.PROPERTY_MODIFIED;
+import static org.apache.usergrid.persistence.Schema.PROPERTY_NAME;
+import static org.apache.usergrid.persistence.Schema.PROPERTY_TIMESTAMP;
+import static org.apache.usergrid.persistence.Schema.PROPERTY_TYPE;
+import static org.apache.usergrid.persistence.Schema.PROPERTY_UUID;
+import static org.apache.usergrid.persistence.Schema.TYPE_APPLICATION;
+import static org.apache.usergrid.persistence.Schema.TYPE_ENTITY;
+import static org.apache.usergrid.persistence.Schema.getDefaultSchema;
import org.apache.usergrid.persistence.SimpleEntityRef;
+import static org.apache.usergrid.persistence.SimpleEntityRef.getUuid;
+import static org.apache.usergrid.persistence.SimpleEntityRef.ref;
+import org.apache.usergrid.persistence.SimpleRoleRef;
import org.apache.usergrid.persistence.TypedEntity;
import org.apache.usergrid.persistence.cassandra.ApplicationCF;
+import static org.apache.usergrid.persistence.cassandra.ApplicationCF.APPLICATION_AGGREGATE_COUNTERS;
+import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_COMPOSITE_DICTIONARIES;
+import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_COUNTERS;
+import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_DICTIONARIES;
import org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils;
import org.apache.usergrid.persistence.cassandra.CassandraService;
+import static org.apache.usergrid.persistence.cassandra.CassandraService.ALL_COUNT;
import org.apache.usergrid.persistence.cassandra.ConnectionRefImpl;
import org.apache.usergrid.persistence.cassandra.CounterUtils;
import org.apache.usergrid.persistence.cassandra.GeoIndexManager;
+import static org.apache.usergrid.persistence.cassandra.Serializers.be;
+import static org.apache.usergrid.persistence.cassandra.Serializers.le;
+import static org.apache.usergrid.persistence.cassandra.Serializers.se;
+import static org.apache.usergrid.persistence.cassandra.Serializers.ue;
import org.apache.usergrid.persistence.cassandra.util.TraceParticipant;
import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.collection.EntityCollectionManager;
@@ -73,6 +121,11 @@ import org.apache.usergrid.persistence.exceptions.DuplicateUniquePropertyExistsE
import org.apache.usergrid.persistence.exceptions.EntityNotFoundException;
import org.apache.usergrid.persistence.exceptions.RequiredPropertyNotFoundException;
import org.apache.usergrid.persistence.exceptions.UnexpectedEntityTypeException;
+import org.apache.usergrid.persistence.graph.Edge;
+import org.apache.usergrid.persistence.graph.GraphManager;
+import org.apache.usergrid.persistence.graph.SearchByEdgeType;
+import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdgeType;
+import org.apache.usergrid.persistence.graph.impl.SimpleSearchEdgeType;
import org.apache.usergrid.persistence.index.EntityIndex;
import org.apache.usergrid.persistence.index.IndexScope;
import org.apache.usergrid.persistence.index.impl.IndexScopeImpl;
@@ -80,76 +133,27 @@ import org.apache.usergrid.persistence.index.query.CounterResolution;
import org.apache.usergrid.persistence.index.query.Identifier;
import org.apache.usergrid.persistence.index.query.Query;
import org.apache.usergrid.persistence.index.query.Query.Level;
+import static org.apache.usergrid.persistence.index.query.Query.Level.REFS;
import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.entity.SimpleId;
import org.apache.usergrid.persistence.model.field.Field;
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
+import org.apache.usergrid.persistence.schema.CollectionInfo;
import org.apache.usergrid.utils.ClassUtils;
-import org.apache.usergrid.utils.CompositeUtils;
-import org.apache.usergrid.utils.StringUtils;
-import org.apache.usergrid.utils.UUIDUtils;
-
-import com.netflix.hystrix.exception.HystrixRuntimeException;
-import com.yammer.metrics.annotation.Metered;
-
-import me.prettyprint.hector.api.Keyspace;
-import me.prettyprint.hector.api.beans.ColumnSlice;
-import me.prettyprint.hector.api.beans.CounterRow;
-import me.prettyprint.hector.api.beans.CounterRows;
-import me.prettyprint.hector.api.beans.CounterSlice;
-import me.prettyprint.hector.api.beans.DynamicComposite;
-import me.prettyprint.hector.api.beans.HColumn;
-import me.prettyprint.hector.api.beans.HCounterColumn;
-import me.prettyprint.hector.api.factory.HFactory;
-import me.prettyprint.hector.api.mutation.Mutator;
-import me.prettyprint.hector.api.query.MultigetSliceCounterQuery;
-import me.prettyprint.hector.api.query.QueryResult;
-import me.prettyprint.hector.api.query.SliceCounterQuery;
-import rx.Observable;
-
-import static java.lang.String.CASE_INSENSITIVE_ORDER;
-import static java.util.Arrays.asList;
-
-import static me.prettyprint.hector.api.factory.HFactory.createCounterSliceQuery;
-import static me.prettyprint.hector.api.factory.HFactory.createMutator;
-import static org.apache.commons.lang.StringUtils.capitalize;
-import static org.apache.commons.lang.StringUtils.isBlank;
-import static org.apache.usergrid.corepersistence.CpRelationManager.ALL_TYPES;
-import static org.apache.usergrid.persistence.Schema.COLLECTION_ROLES;
-import static org.apache.usergrid.persistence.Schema.COLLECTION_USERS;
-import static org.apache.usergrid.persistence.Schema.DICTIONARY_PERMISSIONS;
-import static org.apache.usergrid.persistence.Schema.DICTIONARY_ROLENAMES;
-import static org.apache.usergrid.persistence.Schema.DICTIONARY_ROLETIMES;
-import static org.apache.usergrid.persistence.Schema.DICTIONARY_SETS;
-import static org.apache.usergrid.persistence.Schema.PROPERTY_CREATED;
-import static org.apache.usergrid.persistence.Schema.PROPERTY_INACTIVITY;
-import static org.apache.usergrid.persistence.Schema.PROPERTY_MODIFIED;
-import static org.apache.usergrid.persistence.Schema.PROPERTY_NAME;
-import static org.apache.usergrid.persistence.Schema.PROPERTY_TIMESTAMP;
-import static org.apache.usergrid.persistence.Schema.PROPERTY_TYPE;
-import static org.apache.usergrid.persistence.Schema.PROPERTY_UUID;
-import static org.apache.usergrid.persistence.Schema.TYPE_APPLICATION;
-import static org.apache.usergrid.persistence.Schema.TYPE_ENTITY;
-import org.apache.usergrid.persistence.Schema;
-import static org.apache.usergrid.persistence.SimpleEntityRef.getUuid;
-import static org.apache.usergrid.persistence.SimpleEntityRef.ref;
-import org.apache.usergrid.persistence.SimpleRoleRef;
-import static org.apache.usergrid.persistence.cassandra.ApplicationCF.APPLICATION_AGGREGATE_COUNTERS;
-import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_COMPOSITE_DICTIONARIES;
-import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_COUNTERS;
-import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_DICTIONARIES;
-import static org.apache.usergrid.persistence.cassandra.CassandraService.ALL_COUNT;
-import static org.apache.usergrid.persistence.cassandra.Serializers.be;
-import static org.apache.usergrid.persistence.cassandra.Serializers.le;
-import static org.apache.usergrid.persistence.cassandra.Serializers.se;
-import static org.apache.usergrid.persistence.cassandra.Serializers.ue;
-import static org.apache.usergrid.persistence.index.query.Query.Level.REFS;
import static org.apache.usergrid.utils.ClassUtils.cast;
+import org.apache.usergrid.utils.CompositeUtils;
import static org.apache.usergrid.utils.ConversionUtils.bytebuffer;
import static org.apache.usergrid.utils.ConversionUtils.getLong;
import static org.apache.usergrid.utils.ConversionUtils.object;
import static org.apache.usergrid.utils.ConversionUtils.string;
import static org.apache.usergrid.utils.InflectionUtils.singularize;
+import org.apache.usergrid.utils.StringUtils;
+import org.apache.usergrid.utils.UUIDUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.util.Assert;
+import rx.Observable;
+import rx.functions.Action1;
@@ -172,7 +176,7 @@ public class CpEntityManager implements EntityManager {
private CpManagerCache managerCache;
- private ApplicationScope appScope;
+ private ApplicationScope applicationScope;
private CassandraService cass;
@@ -191,7 +195,7 @@ public class CpEntityManager implements EntityManager {
this.managerCache = this.emf.getManagerCache();
this.applicationId = applicationId;
- appScope = this.emf.getApplicationScope( applicationId );
+ applicationScope = this.emf.getApplicationScope( applicationId );
this.cass = this.emf.cass;
this.counterUtils = this.emf.counterUtils;
@@ -200,7 +204,7 @@ public class CpEntityManager implements EntityManager {
this.skipAggregateCounters = false;
- appScope = this.emf.getApplicationScope( applicationId );
+ applicationScope = this.emf.getApplicationScope( applicationId );
}
@@ -226,6 +230,66 @@ public class CpEntityManager implements EntityManager {
return csn.toLowerCase();
}
+ static boolean isCollectionEdgeType( String type ) {
+ return type.endsWith( EDGE_COLL_SUFFIX );
+ }
+
+ static boolean isConnectionEdgeType( String type ) {
+ return type.endsWith( EDGE_CONN_SUFFIX );
+ }
+
+ static public String getConnectionType( String edgeType ) {
+ String[] parts = edgeType.split("\\|");
+ return parts[0];
+ }
+
+ static public String getCollectionEntityType( String edgeType ) {
+ String[] parts = edgeType.split("\\|");
+ return parts[1];
+ }
+
+ static public String getCollectionName( String edgeType ) {
+ String[] parts = edgeType.split("\\|");
+ return parts[0];
+ }
+
+ static public String getConnectedEntityType( String edgeType ) {
+ String[] parts = edgeType.split("\\|");
+ return parts[1];
+ }
+
+ static String getEdgeTypeFromConnectionType( String connectionType, String targetEntityType ) {
+
+ if ( connectionType != null && targetEntityType != null ) {
+ String csn = connectionType + "|" + targetEntityType + "|" + EDGE_CONN_SUFFIX;
+ return csn;
+ }
+
+ if ( connectionType != null ) {
+ // no suffix, this must be a search
+ String csn = connectionType;
+ return csn;
+ }
+
+ return null;
+ }
+
+ static String getEdgeTypeFromCollectionName( String collectionName, String targetEntityType ) {
+
+ if ( collectionName != null && targetEntityType != null ) {
+ String csn = collectionName + "|" + targetEntityType + "|" + EDGE_COLL_SUFFIX;
+ return csn;
+ }
+
+ if ( collectionName != null ) {
+ // no suffix, this must be a search
+ String csn = collectionName;
+ return csn;
+ }
+
+ return null;
+ }
+
@Override
public Entity create( String entityType, Map<String, Object> properties ) throws Exception {
@@ -318,7 +382,7 @@ public class CpEntityManager implements EntityManager {
String collectionName = getCollectionScopeNameFromEntityType( entityRef.getType() );
CollectionScope collectionScope = new CollectionScopeImpl(
- appScope.getApplication(), appScope.getApplication(), collectionName );
+ applicationScope.getApplication(), applicationScope.getApplication(), collectionName );
EntityCollectionManager ecm = managerCache.getEntityCollectionManager( collectionScope );
@@ -405,7 +469,7 @@ public class CpEntityManager implements EntityManager {
String collectionName = getCollectionScopeNameFromEntityType( type );
CollectionScope collectionScope = new CollectionScopeImpl(
- appScope.getApplication(), appScope.getApplication(), collectionName );
+ applicationScope.getApplication(), applicationScope.getApplication(), collectionName );
EntityCollectionManager ecm = managerCache.getEntityCollectionManager( collectionScope );
@@ -471,8 +535,8 @@ public class CpEntityManager implements EntityManager {
// first, update entity index in its own collection scope
CollectionScope collectionScope = new CollectionScopeImpl(
- appScope.getApplication(),
- appScope.getApplication(),
+ applicationScope.getApplication(),
+ applicationScope.getApplication(),
getCollectionScopeNameFromEntityType( entity.getType() ) );
EntityCollectionManager ecm = managerCache.getEntityCollectionManager( collectionScope );
@@ -534,8 +598,8 @@ public class CpEntityManager implements EntityManager {
private Observable deleteAsync( EntityRef entityRef ) throws Exception {
CollectionScope collectionScope = new CollectionScopeImpl(
- appScope.getApplication(),
- appScope.getApplication(),
+ applicationScope.getApplication(),
+ applicationScope.getApplication(),
getCollectionScopeNameFromEntityType( entityRef.getType() ) );
EntityCollectionManager ecm = managerCache.getEntityCollectionManager( collectionScope );
@@ -568,7 +632,7 @@ public class CpEntityManager implements EntityManager {
for ( String coll : collectionNames ) {
IndexScope indexScope = new IndexScopeImpl(
- appScope.getApplication(),
+ applicationScope.getApplication(),
new SimpleId( uuid, ownerType ),
CpEntityManager.getCollectionScopeNameFromCollectionName(coll) );
@@ -581,15 +645,15 @@ public class CpEntityManager implements EntityManager {
// deindex from default index scope
IndexScope defaultIndexScope = new IndexScopeImpl(
- appScope.getApplication(),
- appScope.getApplication(),
+ applicationScope.getApplication(),
+ applicationScope.getApplication(),
getCollectionScopeNameFromEntityType( entityRef.getType() ) );
EntityIndex entityIndex = managerCache.getEntityIndex( defaultIndexScope );
entityIndex.deindex( entity );
IndexScope allTypesIndexScope = new IndexScopeImpl(
- appScope.getApplication(),
- appScope.getApplication(),
+ applicationScope.getApplication(),
+ applicationScope.getApplication(),
ALL_TYPES);
EntityIndex aei = managerCache.getEntityIndex( allTypesIndexScope );
aei.deindex( entity );
@@ -969,13 +1033,13 @@ public class CpEntityManager implements EntityManager {
String collectionName = getCollectionScopeNameFromEntityType( entityRef.getType() );
CollectionScope collectionScope = new CollectionScopeImpl(
- appScope.getApplication(),
- appScope.getApplication(),
+ applicationScope.getApplication(),
+ applicationScope.getApplication(),
collectionName );
IndexScope defaultIndexScope = new IndexScopeImpl(
- appScope.getApplication(),
- appScope.getApplication(),
+ applicationScope.getApplication(),
+ applicationScope.getApplication(),
getCollectionScopeNameFromEntityType( entityRef.getType()) );
EntityCollectionManager ecm = managerCache.getEntityCollectionManager( collectionScope );
@@ -1238,7 +1302,8 @@ public class CpEntityManager implements EntityManager {
ColumnSlice<ByteBuffer, ByteBuffer> results =
cass.getColumns( cass.getApplicationKeyspace( applicationId ), dictionaryCf,
- CassandraPersistenceUtils.key( entity.getUuid(), dictionaryName ), columnNames, be, be );
+ CassandraPersistenceUtils.key( entity.getUuid(), dictionaryName ),
+ columnNames, be, be );
if ( results != null ) {
values = new HashMap<String, Object>();
for ( HColumn<ByteBuffer, ByteBuffer> result : results.getColumns() ) {
@@ -1618,11 +1683,14 @@ public class CpEntityManager implements EntityManager {
Mutator<ByteBuffer> batch = createMutator( cass.getApplicationKeyspace( applicationId ), be );
CassandraPersistenceUtils.addInsertToMutator( batch, ENTITY_DICTIONARIES,
- CassandraPersistenceUtils.key( ownerId, Schema.DICTIONARY_ROLENAMES ), roleName, roleTitle, timestamp );
+ CassandraPersistenceUtils.key( ownerId, Schema.DICTIONARY_ROLENAMES ),
+ roleName, roleTitle, timestamp );
CassandraPersistenceUtils.addInsertToMutator( batch, ENTITY_DICTIONARIES,
- CassandraPersistenceUtils.key( ownerId, Schema.DICTIONARY_ROLETIMES ), roleName, inactivity, timestamp );
+ CassandraPersistenceUtils.key( ownerId, Schema.DICTIONARY_ROLETIMES ),
+ roleName, inactivity, timestamp );
CassandraPersistenceUtils.addInsertToMutator( batch, ENTITY_DICTIONARIES,
- CassandraPersistenceUtils.key( ownerId, DICTIONARY_SETS ), Schema.DICTIONARY_ROLENAMES, null, timestamp );
+ CassandraPersistenceUtils.key( ownerId, DICTIONARY_SETS ),
+ Schema.DICTIONARY_ROLENAMES, null, timestamp );
CassandraPersistenceUtils.batchExecute( batch, CassandraService.RETRY_COUNT );
@@ -1662,13 +1730,15 @@ public class CpEntityManager implements EntityManager {
private Object getRolePermissionsKey( String roleName ) {
- return CassandraPersistenceUtils.key( SimpleRoleRef.getIdForRoleName( roleName ), DICTIONARY_PERMISSIONS );
+ return CassandraPersistenceUtils.key( SimpleRoleRef.getIdForRoleName( roleName ),
+ DICTIONARY_PERMISSIONS );
}
private Object getRolePermissionsKey( UUID groupId, String roleName ) {
try {
- return CassandraPersistenceUtils.key( getGroupRoleRef( groupId, roleName ).getUuid(), DICTIONARY_PERMISSIONS );
+ return CassandraPersistenceUtils.key( getGroupRoleRef( groupId, roleName ).getUuid(),
+ DICTIONARY_PERMISSIONS );
} catch ( Exception e ) {
logger.error("Error creating role key for uuid {} and role {}", groupId, roleName );
return null;
@@ -2485,8 +2555,8 @@ public class CpEntityManager implements EntityManager {
// prepare to write and index Core Persistence Entity into default scope
CollectionScope collectionScope = new CollectionScopeImpl(
- appScope.getApplication(),
- appScope.getApplication(),
+ applicationScope.getApplication(),
+ applicationScope.getApplication(),
getCollectionScopeNameFromEntityType( eType ) );
EntityCollectionManager ecm = managerCache.getEntityCollectionManager( collectionScope );
@@ -2530,8 +2600,8 @@ public class CpEntityManager implements EntityManager {
// Index CP entity into default collection scope
IndexScope defaultIndexScope = new IndexScopeImpl(
- appScope.getApplication(),
- appScope.getApplication(),
+ applicationScope.getApplication(),
+ applicationScope.getApplication(),
CpEntityManager.getCollectionScopeNameFromEntityType( entity.getType() ) );
EntityIndex ei = managerCache.getEntityIndex( defaultIndexScope );
ei.index( cpEntity );
@@ -2644,16 +2714,18 @@ public class CpEntityManager implements EntityManager {
if ( !removeFromDictionary ) {
// Set the new value
- elementCoValue = CassandraPersistenceUtils.toStorableBinaryValue( elementCoValue, !entityHasDictionary );
+ elementCoValue = CassandraPersistenceUtils.toStorableBinaryValue(
+ elementCoValue, !entityHasDictionary );
- CassandraPersistenceUtils.addInsertToMutator(
- batch, dictionary_cf, CassandraPersistenceUtils.key( entity.getUuid(), dictionaryName ),
+ CassandraPersistenceUtils.addInsertToMutator( batch, dictionary_cf,
+ CassandraPersistenceUtils.key( entity.getUuid(), dictionaryName ),
entityHasDictionary
? elementValue : asList( elementValue ), elementCoValue, timestamp );
if ( !entityHasDictionary ) {
CassandraPersistenceUtils.addInsertToMutator( batch, ENTITY_DICTIONARIES,
- CassandraPersistenceUtils.key( entity.getUuid(), DICTIONARY_SETS ), dictionaryName, null, timestamp );
+ CassandraPersistenceUtils.key( entity.getUuid(), DICTIONARY_SETS ),
+ dictionaryName, null, timestamp );
}
}
else {
@@ -2749,14 +2821,15 @@ public class CpEntityManager implements EntityManager {
// refresh this Entity Manager's application's index
IndexScope indexScope = new IndexScopeImpl(
- appScope.getApplication(), appScope.getApplication(), "dummy" );
+ applicationScope.getApplication(), applicationScope.getApplication(), "dummy" );
EntityIndex ei = managerCache.getEntityIndex( indexScope );
ei.refresh();
}
- public static org.apache.usergrid.persistence.model.entity.Entity entityToCpEntity( Entity entity, UUID importId ) {
+ public static org.apache.usergrid.persistence.model.entity.Entity
+ entityToCpEntity( Entity entity, UUID importId ) {
UUID uuid = importId != null ? importId : entity.getUuid();
@@ -2777,6 +2850,216 @@ public class CpEntityManager implements EntityManager {
public void flushManagerCaches() {
managerCache.flush();
}
+
+
+ /**
+ * Completely reindex the application associated with this EntityManager.
+ */
+ public void reindex( EntityManagerFactory.ProgressObserver po ) throws Exception {
+ indexEntityConnectionsAndCollections( getApplication(), po );
+ }
+
+
+ /**
+ * Recursively index (or reindex) all of the collections and connections of a
+ * specified entity, and all of the collected and connected entities as well.
+ */
+ private void indexEntityConnectionsAndCollections(
+ final EntityRef entity, final EntityManagerFactory.ProgressObserver po ) {
+
+ final GraphManager gm = managerCache.getGraphManager(applicationScope);
+
+ final Id fromEntityId = new SimpleId( entity.getUuid(), entity.getType() );
+
+ logger.debug("Loading edges types from {}:{}\n scope {}:{}",
+ new Object[] { entity.getType(), entity.getUuid(),
+ applicationScope.getApplication().getType(),
+ applicationScope.getApplication().getUuid() } );
+
+ Observable<String> edgeTypes = gm.getEdgeTypesFromSource(
+ new SimpleSearchEdgeType( fromEntityId, null , null ));
+
+ edgeTypes.forEach( new Action1<String>() {
+
+ @Override
+ public void call( final String edgeType ) {
+
+ logger.debug("Loading edges of edgeType {} from {}:{}\n scope {}:{}",
+ new Object[] { edgeType, entity.getType(), entity.getUuid(),
+ applicationScope.getApplication().getType(),
+ applicationScope.getApplication().getUuid() } );
+
+ Observable<Edge> edges = gm.loadEdgesFromSource( new SimpleSearchByEdgeType(
+ fromEntityId, edgeType, Long.MAX_VALUE,
+ SearchByEdgeType.Order.DESCENDING, null ));
+
+ edges.forEach( new Action1<Edge>() {
+
+ @Override
+ public void call( Edge edge ) {
+
+ if ( isCollectionEdgeType( edge.getType() )) {
+
+ String collName = getCollectionName(edgeType);
+ String memberType = getCollectionEntityType(edgeType);
+
+ CollectionScope collScope = new CollectionScopeImpl(
+ applicationScope.getApplication(),
+ applicationScope.getApplication(),
+ CpEntityManager.getCollectionScopeNameFromCollectionName(collName));
+ EntityCollectionManager collMgr =
+ managerCache.getEntityCollectionManager(collScope);
+
+ org.apache.usergrid.persistence.model.entity.Entity collEntity =
+ collMgr.load( edge.getTargetNode()).toBlockingObservable().last();
+
+ CollectionScope memberScope = new CollectionScopeImpl(
+ applicationScope.getApplication(),
+ applicationScope.getApplication(),
+ CpEntityManager.getCollectionScopeNameFromEntityType(memberType));
+ EntityCollectionManager memberMgr =
+ managerCache.getEntityCollectionManager(memberScope);
+
+ org.apache.usergrid.persistence.model.entity.Entity memberEntity =
+ memberMgr.load( edge.getTargetNode()).toBlockingObservable().last();
+
+ indexEntityIntoCollections( collEntity, memberEntity, collName, true );
+
+ EntityRef ref = new SimpleEntityRef(
+ memberEntity.getId().getType(), memberEntity.getId().getUuid());
+ po.onProgress( entity, ref, edge.getType());
+
+ // recursion
+ indexEntityConnectionsAndCollections( new SimpleEntityRef(
+ memberEntity.getId().getType(), memberEntity.getId().getUuid()),po);
+
+ } else if ( isConnectionEdgeType( edge.getType() )) {
+
+ String connType = getConnectionType(edgeType);
+ String targetEntityType = getConnectedEntityType(edgeType);
+ String sourceEntityType = entity.getType();
+
+ CollectionScope sourceScope = new CollectionScopeImpl(
+ applicationScope.getApplication(),
+ applicationScope.getApplication(),
+ CpEntityManager.getCollectionScopeNameFromEntityType(sourceEntityType));
+ EntityCollectionManager sourceEcm =
+ managerCache.getEntityCollectionManager(sourceScope);
+
+ org.apache.usergrid.persistence.model.entity.Entity sourceEntity =
+ sourceEcm.load( edge.getTargetNode()).toBlockingObservable().last();
+
+ CollectionScope targetScope = new CollectionScopeImpl(
+ applicationScope.getApplication(),
+ applicationScope.getApplication(),
+ CpEntityManager.getCollectionScopeNameFromEntityType(targetEntityType));
+ EntityCollectionManager targetEcm =
+ managerCache.getEntityCollectionManager(targetScope);
+
+ org.apache.usergrid.persistence.model.entity.Entity targetEntity =
+ targetEcm.load( edge.getTargetNode() ).toBlockingObservable().last();
+
+ indexEntityIntoConnection(
+ sourceEntity, targetEntity, targetEntityType, connType );
+
+ EntityRef ref = new SimpleEntityRef(
+ targetEntity.getId().getType(), targetEntity.getId().getUuid());
+ po.onProgress( entity, ref, edge.getType());
+
+ // recursion
+ indexEntityConnectionsAndCollections( new SimpleEntityRef(
+ targetEntity.getId().getType(), targetEntity.getId().getUuid()),po);
+ }
+ }
+
+ }); // end foreach on edges
+
+ }
+
+ }); // end foreach on edgeTypes
+
+ }
+
+
+ private void indexEntityIntoCollections(
+ org.apache.usergrid.persistence.model.entity.Entity collectionEntity,
+ org.apache.usergrid.persistence.model.entity.Entity memberEntity,
+ String collName,
+ boolean connectBack ) {
+
+ logger.debug("Indexing into collections {} {}:{} member {}:{}", new Object[] {
+ collName, collectionEntity.getId().getType(), collectionEntity.getId().getUuid(),
+ memberEntity.getId().getType(), memberEntity.getId().getUuid() });
+
+ indexEntityIntoCollection( collectionEntity, memberEntity, collName );
+
+ CollectionInfo collection = getDefaultSchema()
+ .getCollection( memberEntity.getId().getType(), collName);
+
+ if (connectBack && collection != null && collection.getLinkedCollection() != null) {
+ indexEntityIntoCollections(
+ memberEntity, collectionEntity, collection.getLinkedCollection(), false );
+ }
+ }
+
+
+ void indexEntityIntoConnection(
+ org.apache.usergrid.persistence.model.entity.Entity sourceEntity,
+ org.apache.usergrid.persistence.model.entity.Entity targetEntity,
+ String targetEntityType,
+ String connType ) {
+
+ logger.debug("Indexing into connection {} source {}:{} target {}:{}", new Object[] {
+ connType, sourceEntity.getId().getType(), sourceEntity.getId().getUuid(),
+ targetEntity.getId().getType(), targetEntity.getId().getUuid() });
+
+ // Index the new connection in app|source|type context
+ IndexScope indexScope = new IndexScopeImpl(
+ applicationScope.getApplication(),
+ sourceEntity.getId(),
+ CpEntityManager.getConnectionScopeName(targetEntityType, connType));
+ EntityIndex ei = managerCache.getEntityIndex(indexScope);
+ ei.index(targetEntity);
+
+ // Index the new connection in app|scope|all-types context
+ IndexScope allTypesIndexScope = new IndexScopeImpl(
+ applicationScope.getApplication(),
+ sourceEntity.getId(),
+ ALL_TYPES);
+ EntityIndex aei = managerCache.getEntityIndex(allTypesIndexScope);
+ aei.index(targetEntity);
+ }
+
+
+ void indexEntityIntoCollection(
+ org.apache.usergrid.persistence.model.entity.Entity collectionEntity,
+ org.apache.usergrid.persistence.model.entity.Entity memberEntity,
+ String collName ) {
+
+ // index member into entity collection | type scope
+ IndexScope collectionIndexScope = new IndexScopeImpl(
+ applicationScope.getApplication(),
+ collectionEntity.getId(),
+ CpEntityManager.getCollectionScopeNameFromCollectionName(collName));
+ EntityIndex collectionIndex = managerCache.getEntityIndex(collectionIndexScope);
+ collectionIndex.index(memberEntity);
+
+ // index member into entity | all-types scope
+ IndexScope entityAllTypesScope = new IndexScopeImpl(
+ applicationScope.getApplication(),
+ collectionEntity.getId(),
+ ALL_TYPES);
+ EntityIndex entityAllCollectionIndex = managerCache.getEntityIndex(entityAllTypesScope);
+ entityAllCollectionIndex.index(memberEntity);
+
+ // index member into application | all-types scope
+ IndexScope appAllTypesScope = new IndexScopeImpl(
+ applicationScope.getApplication(),
+ applicationScope.getApplication(),
+ ALL_TYPES);
+ EntityIndex allCollectionIndex = managerCache.getEntityIndex(appAllTypesScope);
+ allCollectionIndex.index(memberEntity);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1aa04a71/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index 5ca9536..168c1bf 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -24,47 +24,40 @@ import static java.lang.String.CASE_INSENSITIVE_ORDER;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
-import java.util.Set;
import java.util.TreeMap;
import java.util.UUID;
import org.apache.commons.lang.StringUtils;
+import org.apache.usergrid.persistence.AbstractEntity;
import org.apache.usergrid.persistence.DynamicEntity;
+import org.apache.usergrid.persistence.Entity;
+import org.apache.usergrid.persistence.EntityFactory;
import org.apache.usergrid.persistence.EntityManager;
import org.apache.usergrid.persistence.EntityManagerFactory;
import org.apache.usergrid.persistence.Results;
-import static org.apache.usergrid.persistence.Schema.PROPERTY_CREATED;
import static org.apache.usergrid.persistence.Schema.PROPERTY_NAME;
-import static org.apache.usergrid.persistence.Schema.PROPERTY_UUID;
import static org.apache.usergrid.persistence.Schema.TYPE_APPLICATION;
import org.apache.usergrid.persistence.cassandra.CassandraService;
import org.apache.usergrid.persistence.cassandra.CounterUtils;
import org.apache.usergrid.persistence.cassandra.Setup;
import org.apache.usergrid.persistence.collection.CollectionScope;
-import org.apache.usergrid.persistence.collection.EntityCollectionManager;
import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
import org.apache.usergrid.persistence.entities.Application;
import org.apache.usergrid.persistence.exceptions.ApplicationAlreadyExistsException;
-import org.apache.usergrid.persistence.exceptions.DuplicateUniquePropertyExistsException;
+import org.apache.usergrid.persistence.graph.Edge;
import org.apache.usergrid.persistence.graph.GraphManager;
import org.apache.usergrid.persistence.graph.GraphManagerFactory;
-import org.apache.usergrid.persistence.graph.impl.SimpleSearchEdgeType;
-import org.apache.usergrid.persistence.index.EntityIndex;
+import org.apache.usergrid.persistence.graph.SearchByEdgeType;
+import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdgeType;
import org.apache.usergrid.persistence.index.EntityIndexFactory;
import org.apache.usergrid.persistence.index.IndexScope;
-import org.apache.usergrid.persistence.index.query.CandidateResult;
import org.apache.usergrid.persistence.index.impl.IndexScopeImpl;
-import org.apache.usergrid.persistence.index.query.CandidateResults;
import org.apache.usergrid.persistence.index.query.Query;
-import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.entity.SimpleId;
-import org.apache.usergrid.persistence.model.field.Field;
-import org.apache.usergrid.persistence.model.field.LongField;
-import org.apache.usergrid.persistence.model.field.StringField;
-import org.apache.usergrid.persistence.model.field.UUIDField;
+import org.apache.usergrid.utils.UUIDUtils;
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -93,7 +86,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
// The System Application where we store app and org metadata
public static final UUID SYSTEM_APP_ID =
UUID.fromString("b6768a08-b5d5-11e3-a495-10ddb1de66c3");
-
+
public static final UUID MANAGEMENT_APPLICATION_ID =
UUID.fromString("b6768a08-b5d5-11e3-a495-11ddb1de66c8");
@@ -101,44 +94,6 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
UUID.fromString("b6768a08-b5d5-11e3-a495-11ddb1de66c9");
- @Deprecated // use system app for these in future
- public static final String SYSTEM_APPS_TYPE = "zzzappszzz";
-
- @Deprecated
- public static final String SYSTEM_ORGS_TYPE = "zzzorgszzz";
-
- @Deprecated
- public static final String SYSTEM_PROPS_TYPE = "zzzpropszzz";
-
- @Deprecated // use system app for these in future
- private static final Id systemAppId =
- new SimpleId( SYSTEM_APP_ID, SYSTEM_APPS_TYPE );
-
- @Deprecated
- public static final CollectionScope SYSTEM_APPS_SCOPE =
- new CollectionScopeImpl( systemAppId, systemAppId, SYSTEM_APPS_TYPE );
-
- @Deprecated
- public static final IndexScope SYSTEM_APPS_INDEX_SCOPE =
- new IndexScopeImpl( systemAppId, systemAppId, SYSTEM_APPS_TYPE);
-
- @Deprecated
- public static final CollectionScope SYSTEM_ORGS_SCOPE =
- new CollectionScopeImpl( systemAppId, systemAppId, SYSTEM_ORGS_TYPE);
-
- @Deprecated
- public static final IndexScope SYSTEM_ORGS_INDEX_SCOPE =
- new IndexScopeImpl( systemAppId, systemAppId, SYSTEM_ORGS_TYPE);
-
- @Deprecated
- public static final CollectionScope SYSTEM_PROPS_SCOPE =
- new CollectionScopeImpl( systemAppId, systemAppId, SYSTEM_PROPS_TYPE);
-
- @Deprecated
- public static final IndexScope SYSTEM_PROPS_INDEX_SCOPE =
- new IndexScopeImpl( systemAppId, systemAppId, SYSTEM_PROPS_TYPE);
-
-
// cache of already instantiated entity managers
private LoadingCache<UUID, EntityManager> entityManagers
= CacheBuilder.newBuilder().maximumSize(100).build(new CacheLoader<UUID, EntityManager>() {
@@ -168,21 +123,28 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
logger.warn("NOTE: Counters have been disabled by configuration...");
}
- // if system app does have apps, orgs and props then populate it
+ }
+
+
+ private void init() {
+
+ EntityManager em = getEntityManager(SYSTEM_APP_ID);
+
try {
- EntityManager em = getEntityManager(SYSTEM_APP_ID);
- Results orgs = em.searchCollection(em.getApplicationRef(), "organizations", null);
- if (orgs.isEmpty()) {
- populateSystemAppsFromEs();
- populateSystemOrgsFromEs();
- populateSystemPropsFromEs();
+ if ( em.getApplication() == null ) {
+ logger.info("Creating system application");
+ Map sysAppProps = new HashMap<String, Object>();
+ sysAppProps.put( PROPERTY_NAME, "systemapp");
+ em.create(SYSTEM_APP_ID, TYPE_APPLICATION, sysAppProps );
+ em.getApplication();
+ em.refreshIndex();
}
} catch (Exception ex) {
- throw new RuntimeException("Fatal error migrating data", ex);
+ throw new RuntimeException("Fatal error creating system application", ex);
}
}
-
+
public CpManagerCache getManagerCache() {
@@ -267,7 +229,10 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
public UUID initializeApplication( String organizationName, UUID applicationId, String name,
Map<String, Object> properties ) throws Exception {
- String appName = buildAppName( organizationName, name );
+
+ EntityManager em = getEntityManager(SYSTEM_APP_ID);
+
+ final String appName = buildAppName( organizationName, name );
// check for pre-existing application
if ( lookupApplication( appName ) != null ) {
@@ -278,58 +243,38 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
UUID orgUuid = lookupOrganization( organizationName );
if ( orgUuid == null ) {
-
- // organization does not exist, create it.
- Entity orgInfoEntity = new Entity(generateOrgId( UUIDGenerator.newTimeUUID() ));
- orgUuid = orgInfoEntity.getId().getUuid();
-
- long timestamp = System.currentTimeMillis();
- orgInfoEntity.setField( new LongField( PROPERTY_CREATED, (long)(timestamp / 1000)));
- orgInfoEntity.setField( new StringField( PROPERTY_NAME, name ));
- orgInfoEntity.setField( new UUIDField( PROPERTY_UUID, orgUuid ));
-
- EntityCollectionManager ecm = getManagerCache()
- .getEntityCollectionManager( SYSTEM_ORGS_SCOPE );
- EntityIndex eci = getManagerCache()
- .getEntityIndex( SYSTEM_ORGS_INDEX_SCOPE );
-
- orgInfoEntity = ecm.write( orgInfoEntity ).toBlockingObservable().last();
- eci.index( orgInfoEntity );
- eci.refresh();
+ // create new org because the specified one does not exist
+ final String orgName = organizationName;
+ Entity orgInfo = em.create("organization", new HashMap<String, Object>() {{
+ put( PROPERTY_NAME, orgName );
+ }});
+ em.refreshIndex();
+ orgUuid = orgInfo.getUuid();
}
+ // create appinfo entry in the system app
+ final UUID appId = applicationId;
+ final UUID orgId = orgUuid;
+ Map<String, Object> appInfoMap = new HashMap<String, Object>() {{
+ put( PROPERTY_NAME, appName );
+ put( "applicationUuid", appId );
+ put( "organizationUuid", orgId );
+ }};
+ Entity appInfo = em.create( "appinfo", appInfoMap );
+ em.refreshIndex();
+
+ // create application entity
if ( properties == null ) {
properties = new TreeMap<String, Object>( CASE_INSENSITIVE_ORDER );
}
properties.put( PROPERTY_NAME, appName );
+ EntityManager appEm = getEntityManager( applicationId );
+ appEm.create( applicationId, TYPE_APPLICATION, properties );
+ appEm.resetRoles();
+ appEm.refreshIndex();
- Entity appInfoEntity = new Entity( generateApplicationId( applicationId ));
-
- long timestamp = System.currentTimeMillis();
- appInfoEntity.setField( new LongField( PROPERTY_CREATED, (long)(timestamp / 1000)));
- appInfoEntity.setField( new StringField( PROPERTY_NAME, name ));
- appInfoEntity.setField( new UUIDField( "applicationUuid", applicationId ));
- appInfoEntity.setField( new UUIDField( "organizationUuid", orgUuid ));
-
- // create app in system app scope
- {
- EntityCollectionManager ecm = getManagerCache()
- .getEntityCollectionManager(SYSTEM_APPS_SCOPE );
- EntityIndex eci = getManagerCache()
- .getEntityIndex( SYSTEM_APPS_INDEX_SCOPE );
-
- appInfoEntity = ecm.write( appInfoEntity ).toBlockingObservable().last();
- eci.index( appInfoEntity );
- eci.refresh();
- }
-
- // create app in its own scope
- EntityManager em = getEntityManager( applicationId );
- em.create( applicationId, TYPE_APPLICATION, properties );
- em.resetRoles();
- em.refreshIndex();
-
+ logger.info("Initialized application {}", appName );
return applicationId;
}
@@ -337,8 +282,8 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
public ApplicationScope getApplicationScope( UUID applicationId ) {
// We can always generate a scope, it doesn't matter if the application exists yet or not.
-
- final ApplicationScopeImpl scope = new ApplicationScopeImpl( generateApplicationId( applicationId ) );
+ final ApplicationScopeImpl scope =
+ new ApplicationScopeImpl( generateApplicationId( applicationId ) );
return scope;
}
@@ -348,39 +293,40 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
public UUID importApplication(
String organization, UUID applicationId,
String name, Map<String, Object> properties) throws Exception {
+
throw new UnsupportedOperationException("Not supported yet.");
}
public UUID lookupOrganization( String name) throws Exception {
+ init();
Query q = Query.fromQL(PROPERTY_NAME + " = '" + name + "'");
-
- EntityIndex ei = getManagerCache().getEntityIndex( SYSTEM_ORGS_INDEX_SCOPE );
- CandidateResults results = ei.search( q );
+ EntityManager em = getEntityManager(SYSTEM_APP_ID);
+ Results results = em.searchCollection( em.getApplicationRef(), "organizations", q);
if ( results.isEmpty() ) {
return null;
}
- return results.iterator().next().getId().getUuid();
+ return results.iterator().next().getUuid();
}
@Override
- public UUID lookupApplication( String name) throws Exception {
+ public UUID lookupApplication( String name ) throws Exception {
+ init();
Query q = Query.fromQL( PROPERTY_NAME + " = '" + name + "'");
- EntityIndex ei = getManagerCache().getEntityIndex( SYSTEM_APPS_INDEX_SCOPE );
-
- CandidateResults results = ei.search( q );
+ EntityManager em = getEntityManager(SYSTEM_APP_ID);
+ Results results = em.searchCollection( em.getApplicationRef(), "appinfos", q);
if ( results.isEmpty() ) {
return null;
}
- return results.iterator().next().getId().getUuid();
+ return results.iterator().next().getUuid();
}
@@ -388,45 +334,47 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
@Metered(group = "core", name = "EntityManagerFactory_getApplication")
public Map<String, UUID> getApplications() throws Exception {
- EntityCollectionManager em = getManagerCache()
- .getEntityCollectionManager(SYSTEM_APPS_SCOPE );
- EntityIndex ei = getManagerCache()
- .getEntityIndex( SYSTEM_APPS_INDEX_SCOPE );
-
Map<String, UUID> appMap = new HashMap<String, UUID>();
- String cursor = null;
- boolean done = false;
+ ApplicationScope appScope = getApplicationScope(SYSTEM_APP_ID);
+ GraphManager gm = managerCache.getGraphManager(appScope);
- while ( !done ) {
+ EntityManager em = getEntityManager(SYSTEM_APP_ID);
+ Application app = em.getApplication();
+ Id fromEntityId = new SimpleId( app.getUuid(), app.getType() );
- Query q = Query.fromQL("select *");
- q.setCursor( cursor );
+ String edgeType = CpEntityManager.getEdgeTypeFromCollectionName("appinfos", "appinfo");
- CandidateResults results = ei.search( q );
- cursor = results.getCursor();
+ logger.debug("getApplications(): Loading edges of edgeType {} from {}:{}",
+ new Object[] { edgeType, fromEntityId.getType(), fromEntityId.getUuid() } );
- Iterator<CandidateResult> iter = results.iterator();
- while ( iter.hasNext() ) {
+ Observable<Edge> edges = gm.loadEdgesFromSource( new SimpleSearchByEdgeType(
+ fromEntityId, edgeType, Long.MAX_VALUE,
+ SearchByEdgeType.Order.DESCENDING, null ));
+
+ Iterator<Edge> iter = edges.toBlockingObservable().getIterator();
+ while ( iter.hasNext() ) {
- CandidateResult cr = iter.next();
- Entity e = em.load( cr.getId() ).toBlockingObservable().last();
+ Edge edge = iter.next();
+ Id targetId = edge.getTargetNode();
- if ( cr.getVersion().compareTo( e.getVersion()) < 0 ) {
- logger.debug("Stale version of Entity uuid:{} type:{}, stale v:{}, latest v:{}",
- new Object[] { cr.getId().getUuid(), cr.getId().getType(),
- cr.getVersion(), e.getVersion()});
- continue;
- }
-
- appMap.put(
- (String)e.getField(PROPERTY_NAME).getValue(),
- (UUID)e.getField("applicationUuid").getValue() );
- }
+ logger.debug("getApplications(): Processing edge from {}:{} to {}:{}", new Object[] {
+ edge.getSourceNode().getType(), edge.getSourceNode().getUuid(),
+ edge.getTargetNode().getType(), edge.getTargetNode().getUuid()
+ });
- if ( cursor == null ) {
- done = true;
- }
+ CollectionScope collScope = new CollectionScopeImpl(
+ appScope.getApplication(),
+ appScope.getApplication(),
+ CpEntityManager.getCollectionScopeNameFromCollectionName("appinfos"));
+
+ org.apache.usergrid.persistence.model.entity.Entity e =
+ managerCache.getEntityCollectionManager( collScope ).load( targetId )
+ .toBlockingObservable().lastOrDefault(null);
+
+ appMap.put(
+ (String)e.getField( PROPERTY_NAME ).getValue(),
+ (UUID)e.getField( "applicationUuid" ).getValue());
}
return appMap;
@@ -442,30 +390,26 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
@Override
public Map<String, String> getServiceProperties() {
- EntityIndex ei = getManagerCache()
- .getEntityIndex( SYSTEM_PROPS_INDEX_SCOPE );
- EntityCollectionManager em = getManagerCache()
- .getEntityCollectionManager( SYSTEM_PROPS_SCOPE );
+ Map<String, String> props = new HashMap<String,String>();
+ EntityManager em = getEntityManager(SYSTEM_APP_ID);
Query q = Query.fromQL("select *");
+ Results results = null;
+ try {
+ results = em.searchCollection( em.getApplicationRef(), "propertymaps", q);
- CandidateResults results = ei.search( q );
-
- if ( results.isEmpty() ) {
- return new HashMap<String,String>();
+ } catch (Exception ex) {
+ logger.error("Error getting system properties", ex);
}
- CandidateResult cr = results.iterator().next();
- Entity propsEntity = em.load( cr.getId() ).toBlockingObservable().last();
-
- Map<String, String> props = new HashMap<String, String>();
-
- // intentionally going only one-level deep into fields and treating all
- // values as strings because that is all we need for service properties.
- for ( Field f : propsEntity.getFields() ) {
- props.put( f.getName(), f.getValue().toString() );
+ if ( results == null || results.isEmpty() ) {
+ return props;
}
+ org.apache.usergrid.persistence.Entity e = results.getEntity();
+ for ( String key : e.getProperties().keySet() ) {
+ props.put( key, props.get(key).toString() );
+ }
return props;
}
@@ -473,30 +417,39 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
@Override
public boolean updateServiceProperties(Map<String, String> properties) {
- EntityCollectionManager em = getManagerCache()
- .getEntityCollectionManager( SYSTEM_PROPS_SCOPE );
- EntityIndex ei = getManagerCache()
- .getEntityIndex( SYSTEM_PROPS_INDEX_SCOPE );
-
+ EntityManager em = getEntityManager(SYSTEM_APP_ID);
Query q = Query.fromQL("select *");
- CandidateResults results = ei.search( q );
- Entity propsEntity;
+ Results results = null;
+ try {
+ results = em.searchCollection( em.getApplicationRef(), "propertymaps", q);
+
+ } catch (Exception ex) {
+ logger.error("Error getting system properties", ex);
+ return false;
+ }
+
+ org.apache.usergrid.persistence.Entity propsEntity = null;
+
if ( !results.isEmpty() ) {
- propsEntity = em.load( results.iterator().next().getId()).toBlockingObservable().last();
+ propsEntity = results.getEntity();
+
} else {
- propsEntity = new Entity( new SimpleId( "properties" ));
- long timestamp = System.currentTimeMillis();
- propsEntity.setField( new LongField( PROPERTY_CREATED, (long)(timestamp / 1000)));
+ propsEntity = EntityFactory.newEntity( UUIDUtils.newTimeUUID(), "propertymap");
}
// intentionally going only one-level deep into fields and treating all
// values as strings because that is all we need for service properties
for ( String key : properties.keySet() ) {
- propsEntity.setField( new StringField(key, properties.get(key)) );
+ propsEntity.setProperty( key, properties.get(key).toString() );
}
- propsEntity = em.write( propsEntity ).toBlockingObservable().last();
- ei.index( propsEntity );
+ try {
+ em.update( propsEntity );
+
+ } catch (Exception ex) {
+ logger.error("Error updating service properties", ex);
+ return false;
+ }
return true;
}
@@ -513,27 +466,34 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
@Override
public boolean deleteServiceProperty(String name) {
- EntityCollectionManager em = getManagerCache().getEntityCollectionManager( SYSTEM_PROPS_SCOPE );
- EntityIndex ei = getManagerCache().getEntityIndex( SYSTEM_PROPS_INDEX_SCOPE );
-
+ EntityManager em = getEntityManager(SYSTEM_APP_ID);
Query q = Query.fromQL("select *");
- CandidateResults results = ei.search( q );
-
- Entity propsEntity = em.load(
- results.iterator().next().getId() ).toBlockingObservable().last();
+ Results results = null;
+ try {
+ results = em.searchCollection( em.getApplicationRef(), "propertymaps", q);
- if ( propsEntity == null ) {
- return false; // nothing to delete
+ } catch (Exception ex) {
+ logger.error("Error getting service property for delete of property: " + name, ex);
+ return false;
}
- if ( propsEntity.getField(name) == null ) {
- return false; // no such field
+ org.apache.usergrid.persistence.Entity propsEntity = null;
+
+ if ( !results.isEmpty() ) {
+ propsEntity = results.getEntity();
+
+ } else {
+ propsEntity = EntityFactory.newEntity( UUIDUtils.newTimeUUID(), "propertymap");
}
- propsEntity.removeField( name );
+ try {
+ ((AbstractEntity)propsEntity).clearDataset( name );
+ em.update( propsEntity );
- propsEntity = em.write( propsEntity ).toBlockingObservable().last();
- ei.index( propsEntity );
+ } catch (Exception ex) {
+ logger.error("Error deleting service property name: " + name, ex);
+ return false;
+ }
return true;
}
@@ -565,15 +525,12 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
return MANAGEMENT_APPLICATION_ID;
}
+
@Override
public UUID getDefaultAppId() {
return DEFAULT_APPLICATION_ID;
}
- private Id generateOrgId(UUID id){
- return new SimpleId( id, "organization" );
- }
-
private Id generateApplicationId(UUID id){
return new SimpleId( id, Application.ENTITY_TYPE );
@@ -593,19 +550,21 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
public void refreshIndex() {
- // refresh factory's indexes, will refresh all three index scopes
- managerCache.getEntityIndex( CpEntityManagerFactory.SYSTEM_APPS_INDEX_SCOPE ).refresh();
-
- // these are unecessary because of above call
- //managerCache.getEntityIndex( CpEntityManagerFactory.SYSTEM_ORGS_INDEX_SCOPE ).refresh();
- //managerCache.getEntityIndex( CpEntityManagerFactory.SYSTEM_PROPS_INDEX_SCOPE ).refresh();
-
// refresh special indexes without calling EntityManager refresh because stack overflow
+
+ // system app
+ IndexScope sscope = new IndexScopeImpl(
+ new SimpleId( SYSTEM_APP_ID, "application"),
+ new SimpleId( SYSTEM_APP_ID, "application"), "dummy");
+ managerCache.getEntityIndex( sscope ).refresh();
+
+ // default app
IndexScope mscope = new IndexScopeImpl(
new SimpleId( getManagementAppId(), "application"),
new SimpleId( getManagementAppId(), "application"), "dummy");
managerCache.getEntityIndex( mscope ).refresh();
+ // management app
IndexScope dscope = new IndexScopeImpl(
new SimpleId( getDefaultAppId(), "application"),
new SimpleId( getDefaultAppId(), "application"), "dummy");
@@ -613,137 +572,36 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
}
- public void rebuildInternalIndexes( ProgressObserver po ) throws Exception {
-
- // get all connections from systems app
-// GraphManager gm = managerCache.getGraphManager( CpEntityManagerFactory.SYSTEM_APPS_SCOPE );
-//
-// Observable<String> edgeTypes = gm.getEdgeTypesFromSource(
-// new SimpleSearchEdgeType( systemAppId, null , null ));
-
- logger.info("Rebuilding system apps index");
- rebuildIndexScope(
- CpEntityManagerFactory.SYSTEM_APPS_SCOPE,
- CpEntityManagerFactory.SYSTEM_APPS_INDEX_SCOPE, po );
+ public void rebuildAllIndexes( ProgressObserver po ) throws Exception {
- logger.info("Rebuilding system orgs index");
- rebuildIndexScope(
- CpEntityManagerFactory.SYSTEM_ORGS_SCOPE,
- CpEntityManagerFactory.SYSTEM_ORGS_INDEX_SCOPE, po );
+ logger.info("\n\nRebuilding all indexes\n");
- logger.info("Rebuilding system props index");
- rebuildIndexScope(
- CpEntityManagerFactory.SYSTEM_PROPS_SCOPE,
- CpEntityManagerFactory.SYSTEM_PROPS_INDEX_SCOPE, po );
+ rebuildInternalIndexes( po );
- logger.info("Rebuilding management application index");
- rebuildApplicationIndex( MANAGEMENT_APPLICATION_ID, po );
+ Map<String, UUID> appMap = getApplications();
- logger.info("Rebuilding default application index");
- rebuildApplicationIndex( DEFAULT_APPLICATION_ID, po );
- }
-
-
- private void rebuildIndexScope( CollectionScope cs, IndexScope is, ProgressObserver po ) {
-
- logger.info("Rebuild index scope for {}:{}:{}", new Object[] {
- cs.getOwner(), cs.getApplication(), cs.getName()
- });
+ logger.info("About to rebuild indexes for {} applications", appMap.keySet().size());
- EntityCollectionManager ecm = managerCache.getEntityCollectionManager( cs );
- EntityIndex ei = managerCache.getEntityIndex( is );
-
- Query q = Query.fromQL("select *");
- CandidateResults results = ei.search( q );
-
- Iterator<CandidateResult> iter = results.iterator();
- while (iter.hasNext()) {
- CandidateResult cr = iter.next();
-
- Entity entity = ecm.load(cr.getId()).toBlockingObservable().last();
-
- if ( cr.getVersion().compareTo( entity.getVersion()) < 0 ) {
- logger.warn(" Ignoring stale version uuid:{} type:{} state v:{} latest v:{}",
- new Object[] {
- cr.getId().getUuid(), cr.getId().getType(),
- cr.getVersion(), entity.getVersion()
- });
-
- } else {
-
- logger.info(" Updating entity type {} with id {} for app {}/{}", new Object[] {
- cr.getId().getType(), cr.getId().getUuid(), cs.getApplication().getUuid()
- });
-
- ei.index(entity);
-
- if ( po != null ) {
- po.onProgress();
- }
-
- }
+ for ( UUID appUuid : appMap.values() ) {
+ rebuildApplicationIndexes( appUuid, po );
}
}
+
-
- public void rebuildApplicationIndex( UUID appId, ProgressObserver po ) throws Exception {
-
- EntityManager em = getEntityManager( appId );
-
- Set<String> collections = em.getApplicationCollections();
-
- logger.debug("For app {} found {} collections: {}", new Object[] {
- appId, collections.size(), collections });
-
- for ( String collection : collections ) {
- rebuildCollectionIndex( appId, collection, po );
- }
+ @Override
+ public void rebuildInternalIndexes(ProgressObserver po) throws Exception {
+ rebuildApplicationIndexes(SYSTEM_APP_ID, po);
}
- public void rebuildCollectionIndex( UUID appId, String collectionName, ProgressObserver po )
- throws Exception {
-
- logger.info( "Reindexing collection: {} for app id: {}", collectionName, appId );
-
+ @Override
+ public void rebuildApplicationIndexes( UUID appId, ProgressObserver po ) throws Exception {
+
+ logger.info("Rebuilding index for application id {}", appId);
EntityManager em = getEntityManager( appId );
Application app = em.getApplication();
- // search for all orgs
-
- Query query = new Query();
- query.setLimit(REBUILD_PAGE_SIZE );
- Results r = null;
-
- do {
-
- r = em.searchCollection( app, collectionName, query );
-
- for ( org.apache.usergrid.persistence.Entity entity : r.getEntities() ) {
-
- logger.info( " Updating Entity name {}, type: {}, id: {} in app id: {}", new Object[] {
- entity.getName(), entity.getType(), entity.getUuid(), appId
- } );
-
- try {
- em.update( entity );
-
- if ( po != null ) {
- po.onProgress();
- }
- }
- catch ( DuplicateUniquePropertyExistsException dupee ) {
- logger.error( " Duplicate property for type: {} with id: {} for app id: {}. "
- + "Property name: {} , value: {}", new Object[] {
- entity.getType(), entity.getUuid(), appId, dupee.getPropertyName(),
- dupee.getPropertyValue()
- } );
- }
- }
-
- query.setCursor( r.getCursor() );
- }
- while ( r != null && r.size() == REBUILD_PAGE_SIZE );
+ ((CpEntityManager)em).reindex( po );
}
@@ -756,142 +614,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
}
}
-
- private void populateSystemOrgsFromEs() throws Exception {
-
- logger.info("Migrating system orgs");
-
- EntityCollectionManager ecm = getManagerCache()
- .getEntityCollectionManager(SYSTEM_ORGS_SCOPE);
- EntityIndex ei = getManagerCache()
- .getEntityIndex( SYSTEM_ORGS_INDEX_SCOPE );
-
- EntityManager systemAppEm = getEntityManager(SYSTEM_APP_ID);
-
- String cursor = null;
- boolean done = false;
-
- while ( !done ) {
-
- Query q = Query.fromQL("select *");
- q.setCursor( cursor );
-
- CandidateResults results = ei.search( q );
- cursor = results.getCursor();
-
- Iterator<CandidateResult> iter = results.iterator();
- while ( iter.hasNext() ) {
-
- CandidateResult cr = iter.next();
- Entity e = ecm.load( cr.getId() ).toBlockingObservable().last();
-
- if ( cr.getVersion().compareTo( e.getVersion()) < 0 ) {
- logger.debug("Stale version of Entity uuid:{} type:{}, stale v:{}, latest v:{}",
- new Object[] { cr.getId().getUuid(), cr.getId().getType(),
- cr.getVersion(), e.getVersion()});
- continue;
- }
-
- Map<String, Object> entityMap = CpEntityMapUtils.toMap( e );
- systemAppEm.create("organization", entityMap );
- }
-
- if ( cursor == null ) {
- done = true;
- }
- }
- }
-
-
- private void populateSystemAppsFromEs() throws Exception {
-
- logger.info("Migrating system apps");
-
- EntityCollectionManager ecm = getManagerCache()
- .getEntityCollectionManager(SYSTEM_APPS_SCOPE );
- EntityIndex ei = getManagerCache()
- .getEntityIndex( SYSTEM_APPS_INDEX_SCOPE );
-
- EntityManager systemAppEm = getEntityManager(SYSTEM_APP_ID);
-
- String cursor = null;
- boolean done = false;
-
- while ( !done ) {
-
- Query q = Query.fromQL("select *");
- q.setCursor( cursor );
-
- CandidateResults results = ei.search( q );
- cursor = results.getCursor();
-
- Iterator<CandidateResult> iter = results.iterator();
- while ( iter.hasNext() ) {
-
- CandidateResult cr = iter.next();
- Entity e = ecm.load( cr.getId() ).toBlockingObservable().last();
-
- if ( cr.getVersion().compareTo( e.getVersion()) < 0 ) {
- logger.debug("Stale version of Entity uuid:{} type:{}, stale v:{}, latest v:{}",
- new Object[] { cr.getId().getUuid(), cr.getId().getType(),
- cr.getVersion(), e.getVersion()});
- continue;
- }
-
- Map<String, Object> entityMap = CpEntityMapUtils.toMap( e );
- systemAppEm.create("application", entityMap );
- }
-
- if ( cursor == null ) {
- done = true;
- }
- }
- }
-
-
- private void populateSystemPropsFromEs() throws Exception {
-
- logger.info("Migrating system props");
-
- EntityCollectionManager ecm = getManagerCache()
- .getEntityCollectionManager(SYSTEM_PROPS_SCOPE );
- EntityIndex ei = getManagerCache()
- .getEntityIndex( SYSTEM_PROPS_INDEX_SCOPE );
-
- EntityManager systemAppEm = getEntityManager(SYSTEM_APP_ID);
-
- String cursor = null;
- boolean done = false;
-
- while ( !done ) {
-
- Query q = Query.fromQL("select *");
- q.setCursor( cursor );
-
- CandidateResults results = ei.search( q );
- cursor = results.getCursor();
-
- Iterator<CandidateResult> iter = results.iterator();
- while ( iter.hasNext() ) {
-
- CandidateResult cr = iter.next();
- Entity e = ecm.load( cr.getId() ).toBlockingObservable().last();
-
- if ( cr.getVersion().compareTo( e.getVersion()) < 0 ) {
- logger.debug("Stale version of Entity uuid:{} type:{}, stale v:{}, latest v:{}",
- new Object[] { cr.getId().getUuid(), cr.getId().getType(),
- cr.getVersion(), e.getVersion()});
- continue;
- }
-
- Map<String, Object> entityMap = CpEntityMapUtils.toMap( e );
- systemAppEm.create("property", entityMap );
- }
-
- if ( cursor == null ) {
- done = true;
- }
- }
+ @Override
+ public void rebuildCollectionIndex(UUID appId, String collection, ProgressObserver po ) {
}
-
}