You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by to...@apache.org on 2014/10/02 07:44:22 UTC
[12/13] git commit: Merge branch 'two-dot-o' of
https://git-wip-us.apache.org/repos/asf/incubator-usergrid into esbatching
Merge branch 'two-dot-o' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into esbatching
Conflicts:
stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/ae9974d3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/ae9974d3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/ae9974d3
Branch: refs/heads/esbatching
Commit: ae9974d38210cdef353820d40d8a59d0f0f4140b
Parents: bba08dd b2b8886
Author: Todd Nine <to...@apache.org>
Authored: Wed Oct 1 23:42:59 2014 -0600
Committer: Todd Nine <to...@apache.org>
Committed: Wed Oct 1 23:42:59 2014 -0600
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 528 +++++++++++++++----
.../corepersistence/CpEntityManagerFactory.java | 499 +++++++-----------
.../corepersistence/CpRelationManager.java | 341 +++++-------
.../usergrid/corepersistence/CpSetup.java | 55 +-
.../HybridEntityManagerFactory.java | 18 +-
.../persistence/EntityManagerFactory.java | 12 +-
.../persistence/cassandra/CassandraService.java | 5 -
.../cassandra/EntityManagerFactoryImpl.java | 54 +-
.../PerformanceEntityRebuildIndexTest.java | 278 ++++++++++
.../cassandra/EntityManagerFactoryImplIT.java | 10 +-
stack/core/src/test/resources/log4j.properties | 13 +-
.../usergrid/persistence/index/IndexFig.java | 7 +
.../index/impl/EsEntityIndexImpl.java | 18 +
.../persistence/index/impl/EsProvider.java | 15 +
.../org/apache/usergrid/tools/IndexRebuild.java | 63 ++-
stack/tools/src/main/resources/log4j.properties | 18 +-
16 files changed, 1202 insertions(+), 732 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ae9974d3/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index a5475e7,38453e1..5ce871b
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@@ -73,8 -121,12 +121,13 @@@ import org.apache.usergrid.persistence.
import org.apache.usergrid.persistence.exceptions.EntityNotFoundException;
import org.apache.usergrid.persistence.exceptions.RequiredPropertyNotFoundException;
import org.apache.usergrid.persistence.exceptions.UnexpectedEntityTypeException;
+ import org.apache.usergrid.persistence.graph.Edge;
+ import org.apache.usergrid.persistence.graph.GraphManager;
+ import org.apache.usergrid.persistence.graph.SearchByEdgeType;
+ import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdgeType;
+ import org.apache.usergrid.persistence.graph.impl.SimpleSearchEdgeType;
import org.apache.usergrid.persistence.index.EntityIndex;
+import org.apache.usergrid.persistence.index.EntityIndexBatch;
import org.apache.usergrid.persistence.index.IndexScope;
import org.apache.usergrid.persistence.index.impl.IndexScopeImpl;
import org.apache.usergrid.persistence.index.query.CounterResolution;
@@@ -586,24 -644,20 +650,24 @@@ public class CpEntityManager implement
}
}
+
+
// deindex from default index scope
IndexScope defaultIndexScope = new IndexScopeImpl(
- appScope.getApplication(),
- appScope.getApplication(),
+ applicationScope.getApplication(),
+ applicationScope.getApplication(),
getCollectionScopeNameFromEntityType( entityRef.getType() ) );
- EntityIndex entityIndex = managerCache.getEntityIndex( defaultIndexScope );
- entityIndex.deindex( entity );
+
+ batch.deindex(defaultIndexScope, entity );
IndexScope allTypesIndexScope = new IndexScopeImpl(
- appScope.getApplication(),
- appScope.getApplication(),
+ applicationScope.getApplication(),
+ applicationScope.getApplication(),
ALL_TYPES);
- EntityIndex aei = managerCache.getEntityIndex( allTypesIndexScope );
- aei.deindex( entity );
+
+ batch.deindex( allTypesIndexScope, entity );
+
+ batch.execute();
decrementEntityCollection( Schema.defaultCollectionName( entityId.getType() ) );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ae9974d3/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index 090a4d6,accf2f8..9a10afd
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@@ -47,23 -46,18 +46,19 @@@ import org.apache.usergrid.persistence.
import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
import org.apache.usergrid.persistence.entities.Application;
import org.apache.usergrid.persistence.exceptions.ApplicationAlreadyExistsException;
- import org.apache.usergrid.persistence.exceptions.DuplicateUniquePropertyExistsException;
+ import org.apache.usergrid.persistence.graph.Edge;
+ import org.apache.usergrid.persistence.graph.GraphManager;
import org.apache.usergrid.persistence.graph.GraphManagerFactory;
+ import org.apache.usergrid.persistence.graph.SearchByEdgeType;
+ import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdgeType;
+import org.apache.usergrid.persistence.index.EntityIndex;
- import org.apache.usergrid.persistence.index.EntityIndexBatch;
import org.apache.usergrid.persistence.index.EntityIndexFactory;
import org.apache.usergrid.persistence.index.IndexScope;
- import org.apache.usergrid.persistence.index.query.CandidateResult;
import org.apache.usergrid.persistence.index.impl.IndexScopeImpl;
- import org.apache.usergrid.persistence.index.query.CandidateResults;
import org.apache.usergrid.persistence.index.query.Query;
- import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.entity.SimpleId;
- import org.apache.usergrid.persistence.model.field.Field;
- import org.apache.usergrid.persistence.model.field.LongField;
- import org.apache.usergrid.persistence.model.field.StringField;
- import org.apache.usergrid.persistence.model.field.UUIDField;
+ import org.apache.usergrid.utils.UUIDUtils;
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@@ -321,11 -299,11 +300,11 @@@ public class CpEntityManagerFactory imp
public UUID lookupOrganization( String name) throws Exception {
+ init();
Query q = Query.fromQL(PROPERTY_NAME + " = '" + name + "'");
-
- EntityIndex ei = getManagerCache().getEntityIndex( SYSTEM_ORGS_INDEX_SCOPE );
- CandidateResults results = ei.search(SYSTEM_ORGS_INDEX_SCOPE, q );
- EntityManager em = getEntityManager(SYSTEM_APP_ID);
- Results results = em.searchCollection( em.getApplicationRef(), "organizations", q);
++ EntityManager em = getEntityManager( SYSTEM_APP_ID );
++ Results results = em.searchCollection( em.getApplicationRef(), "organizations", q );
if ( results.isEmpty() ) {
return null;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ae9974d3/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index 260e429,fba758f..0073eb1
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@@ -27,13 -29,13 +29,12 @@@ import java.util.List
import java.util.Map;
import java.util.Set;
import java.util.UUID;
-import me.prettyprint.hector.api.Keyspace;
-import me.prettyprint.hector.api.beans.DynamicComposite;
-import me.prettyprint.hector.api.beans.HColumn;
-import static me.prettyprint.hector.api.factory.HFactory.createMutator;
-import me.prettyprint.hector.api.mutation.Mutator;
-import static org.apache.usergrid.corepersistence.CpEntityManager.getEdgeTypeFromCollectionName;
-import static org.apache.usergrid.corepersistence.CpEntityManager.getEdgeTypeFromConnectionType;
+
- import org.apache.usergrid.persistence.index.EntityIndexBatch;
+import org.apache.usergrid.utils.UUIDUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.util.Assert;
+
import org.apache.usergrid.persistence.ConnectedEntityRef;
import org.apache.usergrid.persistence.ConnectionRef;
import org.apache.usergrid.persistence.Entity;
@@@ -1108,25 -998,13 +1009,25 @@@ public class CpRelationManager implemen
GraphManager gm = managerCache.getGraphManager(applicationScope);
gm.writeEdge(edge).toBlockingObservable().last();
- final EntityIndex ei = managerCache.getEntityIndex(applicationScope);
- final EntityIndexBatch batch = ei.createBatch();
-
- logger.debug("Wrote edgeType {}\n from {}:{}\n to {}:{}\n scope {}:{}", new Object[] {
- edgeType, cpHeadEntity.getId().getType(), cpHeadEntity.getId().getUuid(),
- targetEntity.getId().getType(), targetEntity.getId().getUuid(),
- applicationScope.getApplication().getType(), applicationScope.getApplication().getUuid()});
+ // Index the new connection in app|source|type context
+ IndexScope indexScope = new IndexScopeImpl(
+ applicationScope.getApplication(),
+ cpHeadEntity.getId(),
+ CpEntityManager.getConnectionScopeName( connectedEntityRef.getType(), connectionType ));
- batch.index( indexScope, targetEntity );
++ EntityIndex ei = managerCache.getEntityIndex(indexScope);
++ ei.index( targetEntity );
+
+ // Index the new connection in app|scope|all-types context
+ IndexScope allTypesIndexScope = new IndexScopeImpl(
+ applicationScope.getApplication(),
+ cpHeadEntity.getId(),
+ ALL_TYPES);
++ EntityIndex aei = managerCache.getEntityIndex(allTypesIndexScope);
++ aei.index( targetEntity );
- ((CpEntityManager)em).indexEntityIntoConnection(
- cpHeadEntity, targetEntity, connectedEntityRef.getType(), connectionType );
+ batch.index( allTypesIndexScope, targetEntity );
+
+ batch.execute();
Keyspace ko = cass.getApplicationKeyspace( applicationId );
Mutator<ByteBuffer> m = createMutator( ko, be );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ae9974d3/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 56316d8,2eb8414..7851afe
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@@ -281,18 -674,32 +281,36 @@@ public class EsEntityIndexImpl implemen
}
- @Override
+ public void refresh() {
+ client.admin().indices().prepareRefresh( indexName ).execute().actionGet();
+ log.debug("Refreshed index: " + indexName);
+ }
+
-
@Override
- public CandidateResults getEntityVersions(Id id) {
+ public CandidateResults getEntityVersions( final IndexScope scope, final Id id ) {
Query query = new Query();
- query.addEqualityFilter(ENTITYID_FIELDNAME,id.getUuid().toString());
- CandidateResults results = search( query );
+ query.addEqualityFilter( ENTITYID_FIELDNAME, id.getUuid().toString() );
+ CandidateResults results = search( scope, query );
return results;
}
+ /**
+ * For testing only.
+ */
+ public void deleteIndex() {
+ AdminClient adminClient = client.admin();
+ DeleteIndexResponse response = adminClient.indices().prepareDelete( indexName ).get();
+ if ( response.isAcknowledged() ) {
+ log.info("Deleted index: " + indexName );
+ } else {
+ log.info("Failed to delete index " + indexName );
+ }
+ }
+
+
+ @Override
+ public void refresh() {
+ client.admin().indices().prepareRefresh( indexName ).execute().actionGet();
+ log.debug( "Refreshed index: " + indexName );
+ }
}