You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by to...@apache.org on 2014/10/02 09:32:32 UTC
[01/13] git commit: Fix for partial update error
Repository: incubator-usergrid
Updated Branches:
refs/heads/two-dot-o b2b8886ef -> 38885fa8e
Fix for partial update error
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/a5b62748
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/a5b62748
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/a5b62748
Branch: refs/heads/two-dot-o
Commit: a5b6274864b11f27c8c95d9f1781a96c2b8b01f6
Parents: 0cc97c6
Author: amuramoto <am...@apigee.com>
Authored: Tue Sep 30 15:37:13 2014 -0700
Committer: amuramoto <am...@apigee.com>
Committed: Tue Sep 30 15:37:13 2014 -0700
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 2 +-
.../apache/usergrid/rest/PartialUpdateTest.java | 50 +++++++++++++-------
2 files changed, 34 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a5b62748/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index b27a61d..5e46d7f 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -505,7 +505,7 @@ public class CpEntityManager implements EntityManager {
logger.debug("About to Write {}:{} version {}", new Object[] {
cpEntity.getId().getType(), cpEntity.getId().getUuid(), cpEntity.getVersion() });
- cpEntity = ecm.write( cpEntity ).toBlockingObservable().last();
+ cpEntity = ecm.update( cpEntity ).toBlockingObservable().last();
logger.debug("Wrote {}:{} version {}", new Object[] {
cpEntity.getId().getType(), cpEntity.getId().getUuid(), cpEntity.getVersion() });
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a5b62748/stack/rest/src/test/java/org/apache/usergrid/rest/PartialUpdateTest.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/PartialUpdateTest.java b/stack/rest/src/test/java/org/apache/usergrid/rest/PartialUpdateTest.java
index 366df14..e150755 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/PartialUpdateTest.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/PartialUpdateTest.java
@@ -23,13 +23,14 @@ import java.io.IOException;
import java.util.LinkedHashMap;
import java.util.Map;
import javax.ws.rs.core.MediaType;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.fail;
+
import org.junit.Rule;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static org.junit.Assert.*;
+
/**
* Partial update test.
@@ -40,15 +41,23 @@ public class PartialUpdateTest extends AbstractRestIT {
@Rule
public TestContextSetup context = new TestContextSetup( this );
+ double latitude=37.772837;
+ double longitude=-122.409895;
+
+ Map<String, Object> geolocation = new LinkedHashMap<String, Object>() {{
+ put("latitude", latitude);
+ put("longitude", longitude);
+ }};
+
@Test
public void testPartialUpdate() throws IOException {
// create user bart
-
Map<String, Object> userProperties = new LinkedHashMap<String, Object>() {{
put( "username", "bart" );
put( "employer", "Brawndo" );
put( "email", "bart@personal-email.example.com" );
+ put( "location", geolocation);
}};
JsonNode userNode = mapper.readTree(
@@ -64,22 +73,29 @@ public class PartialUpdateTest extends AbstractRestIT {
refreshIndex( "test-organization", "test-app" );
- // update user bart passing only an update to his employer
-
- Map<String, Object> updateProperties = new LinkedHashMap<String, Object>() {{
- put( "employer", "Initech" );
- }};
+ // update user bart passing only an update to a property
+ for(int i=1; i<20; i++) {
+ geolocation.put("latitude", latitude += 0.00001);
+ geolocation.put("longitude", longitude += 0.00001);
+ Map<String, Object> updateProperties = new LinkedHashMap<String, Object>() {{
+ put("employer", "Initech");
+ put("location", geolocation);
+ }};
- try {
- JsonNode updatedNode = mapper.readTree(
- resource().path( "/test-organization/test-app/user/" + uuid )
- .queryParam( "access_token", adminAccessToken )
- .accept( MediaType.APPLICATION_JSON )
- .type( MediaType.APPLICATION_JSON )
- .put( String.class, updateProperties ));
+ try {
+ JsonNode updatedNode = mapper.readTree(
+ resource().path("/test-organization/test-app/user/" + uuid)
+ .queryParam("access_token", adminAccessToken)
+ .accept(MediaType.APPLICATION_JSON)
+ .type(MediaType.APPLICATION_JSON)
+ .put(String.class, updateProperties));
+ assertNotNull(updatedNode);
+ assertNotEquals(latitude, updatedNode.get("entities").get(0).get("location").get("latitude"));
+ assertNotEquals(longitude, updatedNode.get("entities").get(0).get("location").get("longitude"));
- } catch ( UniformInterfaceException uie ) {
- fail("Update failed due to: " + uie.getResponse().getEntity(String.class));
+ } catch (UniformInterfaceException uie) {
+ fail("Update failed due to: " + uie.getResponse().getEntity(String.class));
+ }
}
}
}
[13/13] git commit: Removed obsolete test.
Posted by to...@apache.org.
Removed obsolete test.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/38885fa8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/38885fa8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/38885fa8
Branch: refs/heads/two-dot-o
Commit: 38885fa8e1cdf0217369d6699958be0d330f6266
Parents: e005515
Author: Todd Nine <to...@apache.org>
Authored: Thu Oct 2 01:31:09 2014 -0600
Committer: Todd Nine <to...@apache.org>
Committed: Thu Oct 2 01:31:09 2014 -0600
----------------------------------------------------------------------
.../usergrid/corepersistence/CpEntityIndexDeleteListenerTest.java | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/38885fa8/stack/core/src/test/java/org/apache/usergrid/corepersistence/CpEntityIndexDeleteListenerTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/CpEntityIndexDeleteListenerTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/CpEntityIndexDeleteListenerTest.java
index e7675d6..77c62c5 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/CpEntityIndexDeleteListenerTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/CpEntityIndexDeleteListenerTest.java
@@ -25,6 +25,7 @@ import java.util.UUID;
import org.jukito.JukitoRunner;
import org.junit.Before;
+import org.junit.Ignore;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -57,6 +58,7 @@ import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@RunWith( JukitoRunner.class )
+@Ignore("Needs updated")
public class CpEntityIndexDeleteListenerTest {
EntityIndex entityIndex;
CpEntityIndexDeleteListener esEntityIndexDeleteListener;
[08/13] git commit: Merge branch 'two-dot-o' of
https://git-wip-us.apache.org/repos/asf/incubator-usergrid into esbatching
Posted by to...@apache.org.
Merge branch 'two-dot-o' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into esbatching
Conflicts:
stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/ae9974d3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/ae9974d3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/ae9974d3
Branch: refs/heads/two-dot-o
Commit: ae9974d38210cdef353820d40d8a59d0f0f4140b
Parents: bba08dd b2b8886
Author: Todd Nine <to...@apache.org>
Authored: Wed Oct 1 23:42:59 2014 -0600
Committer: Todd Nine <to...@apache.org>
Committed: Wed Oct 1 23:42:59 2014 -0600
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 528 +++++++++++++++----
.../corepersistence/CpEntityManagerFactory.java | 499 +++++++-----------
.../corepersistence/CpRelationManager.java | 341 +++++-------
.../usergrid/corepersistence/CpSetup.java | 55 +-
.../HybridEntityManagerFactory.java | 18 +-
.../persistence/EntityManagerFactory.java | 12 +-
.../persistence/cassandra/CassandraService.java | 5 -
.../cassandra/EntityManagerFactoryImpl.java | 54 +-
.../PerformanceEntityRebuildIndexTest.java | 278 ++++++++++
.../cassandra/EntityManagerFactoryImplIT.java | 10 +-
stack/core/src/test/resources/log4j.properties | 13 +-
.../usergrid/persistence/index/IndexFig.java | 7 +
.../index/impl/EsEntityIndexImpl.java | 18 +
.../persistence/index/impl/EsProvider.java | 15 +
.../org/apache/usergrid/tools/IndexRebuild.java | 63 ++-
stack/tools/src/main/resources/log4j.properties | 18 +-
16 files changed, 1202 insertions(+), 732 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ae9974d3/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index a5475e7,38453e1..5ce871b
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@@ -73,8 -121,12 +121,13 @@@ import org.apache.usergrid.persistence.
import org.apache.usergrid.persistence.exceptions.EntityNotFoundException;
import org.apache.usergrid.persistence.exceptions.RequiredPropertyNotFoundException;
import org.apache.usergrid.persistence.exceptions.UnexpectedEntityTypeException;
+ import org.apache.usergrid.persistence.graph.Edge;
+ import org.apache.usergrid.persistence.graph.GraphManager;
+ import org.apache.usergrid.persistence.graph.SearchByEdgeType;
+ import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdgeType;
+ import org.apache.usergrid.persistence.graph.impl.SimpleSearchEdgeType;
import org.apache.usergrid.persistence.index.EntityIndex;
+import org.apache.usergrid.persistence.index.EntityIndexBatch;
import org.apache.usergrid.persistence.index.IndexScope;
import org.apache.usergrid.persistence.index.impl.IndexScopeImpl;
import org.apache.usergrid.persistence.index.query.CounterResolution;
@@@ -586,24 -644,20 +650,24 @@@ public class CpEntityManager implement
}
}
+
+
// deindex from default index scope
IndexScope defaultIndexScope = new IndexScopeImpl(
- appScope.getApplication(),
- appScope.getApplication(),
+ applicationScope.getApplication(),
+ applicationScope.getApplication(),
getCollectionScopeNameFromEntityType( entityRef.getType() ) );
- EntityIndex entityIndex = managerCache.getEntityIndex( defaultIndexScope );
- entityIndex.deindex( entity );
+
+ batch.deindex(defaultIndexScope, entity );
IndexScope allTypesIndexScope = new IndexScopeImpl(
- appScope.getApplication(),
- appScope.getApplication(),
+ applicationScope.getApplication(),
+ applicationScope.getApplication(),
ALL_TYPES);
- EntityIndex aei = managerCache.getEntityIndex( allTypesIndexScope );
- aei.deindex( entity );
+
+ batch.deindex( allTypesIndexScope, entity );
+
+ batch.execute();
decrementEntityCollection( Schema.defaultCollectionName( entityId.getType() ) );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ae9974d3/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index 090a4d6,accf2f8..9a10afd
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@@ -47,23 -46,18 +46,19 @@@ import org.apache.usergrid.persistence.
import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
import org.apache.usergrid.persistence.entities.Application;
import org.apache.usergrid.persistence.exceptions.ApplicationAlreadyExistsException;
- import org.apache.usergrid.persistence.exceptions.DuplicateUniquePropertyExistsException;
+ import org.apache.usergrid.persistence.graph.Edge;
+ import org.apache.usergrid.persistence.graph.GraphManager;
import org.apache.usergrid.persistence.graph.GraphManagerFactory;
+ import org.apache.usergrid.persistence.graph.SearchByEdgeType;
+ import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdgeType;
+import org.apache.usergrid.persistence.index.EntityIndex;
- import org.apache.usergrid.persistence.index.EntityIndexBatch;
import org.apache.usergrid.persistence.index.EntityIndexFactory;
import org.apache.usergrid.persistence.index.IndexScope;
- import org.apache.usergrid.persistence.index.query.CandidateResult;
import org.apache.usergrid.persistence.index.impl.IndexScopeImpl;
- import org.apache.usergrid.persistence.index.query.CandidateResults;
import org.apache.usergrid.persistence.index.query.Query;
- import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.entity.SimpleId;
- import org.apache.usergrid.persistence.model.field.Field;
- import org.apache.usergrid.persistence.model.field.LongField;
- import org.apache.usergrid.persistence.model.field.StringField;
- import org.apache.usergrid.persistence.model.field.UUIDField;
+ import org.apache.usergrid.utils.UUIDUtils;
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@@ -321,11 -299,11 +300,11 @@@ public class CpEntityManagerFactory imp
public UUID lookupOrganization( String name) throws Exception {
+ init();
Query q = Query.fromQL(PROPERTY_NAME + " = '" + name + "'");
-
- EntityIndex ei = getManagerCache().getEntityIndex( SYSTEM_ORGS_INDEX_SCOPE );
- CandidateResults results = ei.search(SYSTEM_ORGS_INDEX_SCOPE, q );
- EntityManager em = getEntityManager(SYSTEM_APP_ID);
- Results results = em.searchCollection( em.getApplicationRef(), "organizations", q);
++ EntityManager em = getEntityManager( SYSTEM_APP_ID );
++ Results results = em.searchCollection( em.getApplicationRef(), "organizations", q );
if ( results.isEmpty() ) {
return null;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ae9974d3/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index 260e429,fba758f..0073eb1
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@@ -27,13 -29,13 +29,12 @@@ import java.util.List
import java.util.Map;
import java.util.Set;
import java.util.UUID;
-import me.prettyprint.hector.api.Keyspace;
-import me.prettyprint.hector.api.beans.DynamicComposite;
-import me.prettyprint.hector.api.beans.HColumn;
-import static me.prettyprint.hector.api.factory.HFactory.createMutator;
-import me.prettyprint.hector.api.mutation.Mutator;
-import static org.apache.usergrid.corepersistence.CpEntityManager.getEdgeTypeFromCollectionName;
-import static org.apache.usergrid.corepersistence.CpEntityManager.getEdgeTypeFromConnectionType;
+
- import org.apache.usergrid.persistence.index.EntityIndexBatch;
+import org.apache.usergrid.utils.UUIDUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.util.Assert;
+
import org.apache.usergrid.persistence.ConnectedEntityRef;
import org.apache.usergrid.persistence.ConnectionRef;
import org.apache.usergrid.persistence.Entity;
@@@ -1108,25 -998,13 +1009,25 @@@ public class CpRelationManager implemen
GraphManager gm = managerCache.getGraphManager(applicationScope);
gm.writeEdge(edge).toBlockingObservable().last();
- final EntityIndex ei = managerCache.getEntityIndex(applicationScope);
- final EntityIndexBatch batch = ei.createBatch();
-
- logger.debug("Wrote edgeType {}\n from {}:{}\n to {}:{}\n scope {}:{}", new Object[] {
- edgeType, cpHeadEntity.getId().getType(), cpHeadEntity.getId().getUuid(),
- targetEntity.getId().getType(), targetEntity.getId().getUuid(),
- applicationScope.getApplication().getType(), applicationScope.getApplication().getUuid()});
+ // Index the new connection in app|source|type context
+ IndexScope indexScope = new IndexScopeImpl(
+ applicationScope.getApplication(),
+ cpHeadEntity.getId(),
+ CpEntityManager.getConnectionScopeName( connectedEntityRef.getType(), connectionType ));
- batch.index( indexScope, targetEntity );
++ EntityIndex ei = managerCache.getEntityIndex(indexScope);
++ ei.index( targetEntity );
+
+ // Index the new connection in app|scope|all-types context
+ IndexScope allTypesIndexScope = new IndexScopeImpl(
+ applicationScope.getApplication(),
+ cpHeadEntity.getId(),
+ ALL_TYPES);
++ EntityIndex aei = managerCache.getEntityIndex(allTypesIndexScope);
++ aei.index( targetEntity );
- ((CpEntityManager)em).indexEntityIntoConnection(
- cpHeadEntity, targetEntity, connectedEntityRef.getType(), connectionType );
+ batch.index( allTypesIndexScope, targetEntity );
+
+ batch.execute();
Keyspace ko = cass.getApplicationKeyspace( applicationId );
Mutator<ByteBuffer> m = createMutator( ko, be );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ae9974d3/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 56316d8,2eb8414..7851afe
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@@ -281,18 -674,32 +281,36 @@@ public class EsEntityIndexImpl implemen
}
- @Override
+ public void refresh() {
+ client.admin().indices().prepareRefresh( indexName ).execute().actionGet();
+ log.debug("Refreshed index: " + indexName);
+ }
+
-
@Override
- public CandidateResults getEntityVersions(Id id) {
+ public CandidateResults getEntityVersions( final IndexScope scope, final Id id ) {
Query query = new Query();
- query.addEqualityFilter(ENTITYID_FIELDNAME,id.getUuid().toString());
- CandidateResults results = search( query );
+ query.addEqualityFilter( ENTITYID_FIELDNAME, id.getUuid().toString() );
+ CandidateResults results = search( scope, query );
return results;
}
+ /**
+ * For testing only.
+ */
+ public void deleteIndex() {
+ AdminClient adminClient = client.admin();
+ DeleteIndexResponse response = adminClient.indices().prepareDelete( indexName ).get();
+ if ( response.isAcknowledged() ) {
+ log.info("Deleted index: " + indexName );
+ } else {
+ log.info("Failed to delete index " + indexName );
+ }
+ }
+
+
+ @Override
+ public void refresh() {
+ client.admin().indices().prepareRefresh( indexName ).execute().actionGet();
+ log.debug( "Refreshed index: " + indexName );
+ }
}
[02/13] git commit: merged upstream
Posted by to...@apache.org.
merged upstream
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/16d497ed
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/16d497ed
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/16d497ed
Branch: refs/heads/two-dot-o
Commit: 16d497ed29a9c24629a81980dcecf421e773034c
Parents: a5b6274 e5bcbb2
Author: amuramoto <am...@apigee.com>
Authored: Tue Sep 30 15:53:15 2014 -0700
Committer: amuramoto <am...@apigee.com>
Committed: Tue Sep 30 15:53:15 2014 -0700
----------------------------------------------------------------------
.gitignore | 1 +
portal/js/global/ug-service.js | 1100 +++--
portal/js/libs/usergrid.sdk.js | 4264 +++++++++---------
sdks/java/pom.xml | 33 +-
stack/.gitignore | 4 +-
.../main/dist/init_instance/init_rest_server.sh | 38 +-
.../dist/init_instance/install_cassandra.sh | 14 +-
.../dist/init_instance/install_elasticsearch.sh | 32 +-
.../main/dist/init_instance/install_yourkit.sh | 14 +-
stack/awscluster/ugcluster-cf.json | 3 +-
stack/config/README.txt | 4 +-
.../main/resources/usergrid-default.properties | 2 +-
stack/core/pom.xml | 10 -
.../corepersistence/CpEntityManager.java | 12 +-
.../corepersistence/CpEntityManagerFactory.java | 184 +-
.../corepersistence/CpRelationManager.java | 71 +-
.../HybridEntityManagerFactory.java | 10 +
.../apache/usergrid/metrics/MetricsFactory.java | 18 +-
.../usergrid/mq/cassandra/QueueManagerImpl.java | 18 +-
.../mq/cassandra/io/AbstractSearch.java | 172 +-
.../mq/cassandra/io/ConsumerTransaction.java | 17 +-
.../persistence/EntityManagerFactory.java | 4 +
.../cassandra/EntityManagerFactoryImpl.java | 51 +
.../usergrid/persistence/entities/Receipt.java | 20 +-
.../main/resources/usergrid-core-context.xml | 13 +-
.../java/org/apache/usergrid/Application.java | 7 +
.../org/apache/usergrid/CoreApplication.java | 7 +
.../persistence/PerformanceEntityReadTest.java | 173 +
.../persistence/PerformanceEntityWriteTest.java | 170 +
.../cassandra/QueryProcessorTest.java | 2 -
stack/core/src/test/resources/log4j.properties | 1 +
.../impl/EntityCollectionManagerImpl.java | 8 +-
.../mvcc/stage/write/WriteUniqueVerify.java | 2 +-
.../persistence/core/astyanax/CassandraFig.java | 4 +-
.../core/migration/MigrationManagerImpl.java | 6 +
.../graph/impl/GraphManagerImpl.java | 6 +-
.../impl/shard/impl/NodeShardCacheImpl.java | 7 +-
.../impl/shard/NodeShardCacheTest.java | 7 +-
.../index/impl/EsEntityIndexImpl.java | 185 +-
.../persistence/index/impl/EsProvider.java | 1 +
.../persistence/index/impl/EsQueryVistor.java | 151 +-
.../persistence/index/utils/LRUCache2.java | 151 -
.../index/impl/ElasticSearchTest.java | 273 --
.../impl/EntityConnectionIndexImplTest.java | 2 -
.../index/impl/EntityIndexMapUtils.java | 12 +-
.../persistence/index/impl/EntityIndexTest.java | 79 +-
stack/launcher/README.txt | 2 +-
.../resources/usergrid-standalone-context.xml | 2 +-
stack/pom.xml | 2 +-
.../apache/usergrid/rest/SystemResource.java | 32 +-
.../apache/usergrid/rest/AbstractRestIT.java | 6 +-
.../java/org/apache/usergrid/rest/BasicIT.java | 34 +-
.../usergrid/rest/ConcurrentRestITSuite.java | 54 +-
.../apache/usergrid/rest/NotificationsIT.java | 239 +
.../apache/usergrid/rest/PartialUpdateTest.java | 53 +-
.../org/apache/usergrid/rest/RestITSuite.java | 45 +-
.../apache/usergrid/rest/TomcatResource.java | 4 +-
.../ApplicationRequestCounterIT.java | 97 -
.../rest/applications/DevicesResourceIT.java | 87 -
.../collection/BadGrammarQueryTest.java | 79 -
.../collection/CollectionsResourceIT.java | 205 +
.../collection/PagingResourceIT.java | 239 -
.../activities/ActivityResourceIT.java | 188 +
.../collection/activities/AndOrQueryTest.java | 203 -
.../collection/activities/OrderByTest.java | 172 -
.../activities/PagingEntitiesTest.java | 141 -
.../collection/devices/DevicesResourceIT.java | 87 +
.../collection/groups/GeoPagingTest.java | 133 -
.../collection/groups/GroupResourceIT.java | 295 ++
.../collection/paging/PagingEntitiesTest.java | 141 +
.../collection/paging/PagingResourceIT.java | 301 ++
.../users/ConnectionResourceTest.java | 271 ++
.../collection/users/OwnershipResourceIT.java | 379 ++
.../collection/users/PermissionsResourceIT.java | 768 ++++
.../collection/users/RetrieveUsersTest.java | 87 +
.../collection/users/UserResourceIT.java | 1418 ++++++
.../users/extensions/TestResource.java | 51 +
.../events/ApplicationRequestCounterIT.java | 97 +
.../applications/queries/AndOrQueryTest.java | 203 +
.../queries/BadGrammarQueryTest.java | 79 +
.../applications/queries/GeoPagingTest.java | 133 +
.../applications/queries/MatrixQueryTests.java | 202 +
.../rest/applications/queries/OrderByTest.java | 172 +
.../applications/users/ActivityResourceIT.java | 188 -
.../users/CollectionsResourceIT.java | 205 -
.../users/ConnectionResourceTest.java | 271 --
.../applications/users/GroupResourceIT.java | 295 --
.../applications/users/MatrixQueryTests.java | 202 -
.../applications/users/OwnershipResourceIT.java | 379 --
.../users/PermissionsResourceIT.java | 768 ----
.../applications/users/RetrieveUsersTest.java | 87 -
.../rest/applications/users/UserResourceIT.java | 1418 ------
.../users/extensions/TestResource.java | 51 -
.../usergrid/rest/management/AccessTokenIT.java | 350 ++
.../usergrid/rest/management/AdminUsersIT.java | 807 ++++
.../rest/management/ManagementResourceIT.java | 397 +-
.../rest/management/OrganizationsIT.java | 378 ++
.../organizations/OrganizationResourceIT.java | 90 -
.../organizations/OrganizationsResourceIT.java | 322 --
.../rest/management/users/MUUserResourceIT.java | 654 ---
.../UsersOrganizationsResourceIT.java | 72 -
stack/rest/src/test/resources/log4j.properties | 18 +-
.../notifications/ApplicationQueueManager.java | 182 +-
.../notifications/NotificationsService.java | 31 +-
.../notifications/NotificationsTaskManager.java | 33 -
.../services/notifications/QueueJob.java | 4 +-
.../services/notifications/QueueListener.java | 235 +-
.../services/notifications/QueueManager.java | 2 -
.../notifications/SingleQueueTaskManager.java | 115 +-
.../services/notifications/TaskManager.java | 199 -
.../services/notifications/TaskTracker.java | 4 +-
.../services/notifications/gcm/GCMAdapter.java | 61 +-
.../resources/usergrid-services-context.xml | 9 +-
.../AbstractServiceNotificationIT.java | 22 +-
.../apns/NotificationsServiceIT.java | 12 +-
.../gcm/NotificationsServiceIT.java | 9 +-
stack/tools/README.md | 22 +-
.../org/apache/usergrid/tools/IndexRebuild.java | 99 +-
.../org/apache/usergrid/tools/ToolBase.java | 26 +-
119 files changed, 11428 insertions(+), 10653 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/16d497ed/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 5e46d7f,f85ba30..f62c3fd
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@@ -505,7 -503,8 +503,9 @@@ public class CpEntityManager implement
logger.debug("About to Write {}:{} version {}", new Object[] {
cpEntity.getId().getType(), cpEntity.getId().getUuid(), cpEntity.getVersion() });
+ cpEntity = ecm.update( cpEntity ).toBlockingObservable().last();
+ // using ecm.update() here causes Core tests to fail
- cpEntity = ecm.write( cpEntity ).toBlockingObservable().last();
++// cpEntity = ecm.write( cpEntity ).toBlockingObservable().last();
logger.debug("Wrote {}:{} version {}", new Object[] {
cpEntity.getId().getType(), cpEntity.getId().getUuid(), cpEntity.getVersion() });
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/16d497ed/stack/rest/src/test/java/org/apache/usergrid/rest/PartialUpdateTest.java
----------------------------------------------------------------------
diff --cc stack/rest/src/test/java/org/apache/usergrid/rest/PartialUpdateTest.java
index e150755,d685811..c2f3fcb
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/PartialUpdateTest.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/PartialUpdateTest.java
@@@ -23,7 -23,9 +23,13 @@@ import java.io.IOException
import java.util.LinkedHashMap;
import java.util.Map;
import javax.ws.rs.core.MediaType;
++<<<<<<< HEAD
+
++=======
+ import static org.junit.Assert.assertEquals;
+ import static org.junit.Assert.assertNotNull;
+ import static org.junit.Assert.fail;
++>>>>>>> e5bcbb23be66d12e428d817fd8b0b495793a939c
import org.junit.Rule;
import org.junit.Test;
import org.slf4j.Logger;
@@@ -57,7 -50,10 +63,14 @@@ public class PartialUpdateTest extends
put( "username", "bart" );
put( "employer", "Brawndo" );
put( "email", "bart@personal-email.example.com" );
++<<<<<<< HEAD
+ put( "location", geolocation);
++=======
+ put( "location", new LinkedHashMap<String, Object>() {{
+ put("latitude", "37.3638875323994");
+ put("longitude", "-122.12334411518498");
+ }} );
++>>>>>>> e5bcbb23be66d12e428d817fd8b0b495793a939c
}};
JsonNode userNode = mapper.readTree(
@@@ -70,32 -66,63 +83,68 @@@
assertNotNull( userNode );
String uuid = userNode.withArray("entities").get(0).get("uuid").asText();
assertNotNull( uuid );
-
refreshIndex( "test-organization", "test-app" );
- // Update bart's employer without specifying any required fields
- // (with uuid specified in URL)
-
- Map<String, Object> updateProperties = new LinkedHashMap<String, Object>() {{
- put( "employer", "Initech" );
- }};
-
- try {
- mapper.readTree(
- resource().path( "/test-organization/test-app/user/" + uuid )
- .queryParam( "access_token", adminAccessToken )
- .accept( MediaType.APPLICATION_JSON )
- .type( MediaType.APPLICATION_JSON )
- .put( String.class, updateProperties ));
-
- } catch ( UniformInterfaceException uie ) {
- fail("Update failed due to: " + uie.getResponse().getEntity(String.class));
+ // update user bart passing only an update to a property
+ for(int i=1; i<20; i++) {
+ geolocation.put("latitude", latitude += 0.00001);
+ geolocation.put("longitude", longitude += 0.00001);
+ Map<String, Object> updateProperties = new LinkedHashMap<String, Object>() {{
+ put("employer", "Initech");
+ put("location", geolocation);
+ }};
+
+ try {
+ JsonNode updatedNode = mapper.readTree(
+ resource().path("/test-organization/test-app/user/" + uuid)
+ .queryParam("access_token", adminAccessToken)
+ .accept(MediaType.APPLICATION_JSON)
+ .type(MediaType.APPLICATION_JSON)
+ .put(String.class, updateProperties));
+ assertNotNull(updatedNode);
+ assertNotEquals(latitude, updatedNode.get("entities").get(0).get("location").get("latitude"));
+ assertNotEquals(longitude, updatedNode.get("entities").get(0).get("location").get("longitude"));
-
+ } catch (UniformInterfaceException uie) {
+ fail("Update failed due to: " + uie.getResponse().getEntity(String.class));
+ }
}
+ refreshIndex( "test-organization", "test-app" );
+
+ userNode = mapper.readTree(
+ resource().path( "/test-organization/test-app/users/" + uuid )
+ .queryParam( "access_token", adminAccessToken )
+ .accept( MediaType.APPLICATION_JSON )
+ .get( String.class ));
+ assertNotNull( userNode );
+ assertEquals( "Initech", userNode.withArray("entities").get(0).get("employer").asText());
+
+
+ // Update bart's employer without specifying any required fields
+ // (this time with username specified in URL)
+
+ updateProperties = new LinkedHashMap<String, Object>() {{
+ put( "employer", "ACME Corporation" );
+ }};
+
+ try {
+ mapper.readTree(
+ resource().path( "/test-organization/test-app/users/bart")
+ .queryParam( "access_token", adminAccessToken )
+ .accept( MediaType.APPLICATION_JSON )
+ .type( MediaType.APPLICATION_JSON )
+ .put( String.class, updateProperties ));
+
+ } catch ( UniformInterfaceException uie ) {
+ fail("Update failed due to: " + uie.getResponse().getEntity(String.class));
+ }
+ refreshIndex( "test-organization", "test-app" );
+
+ userNode = mapper.readTree(
+ resource().path( "/test-organization/test-app/users/bart" )
+ .queryParam( "access_token", adminAccessToken )
+ .accept( MediaType.APPLICATION_JSON )
+ .get( String.class ));
+ assertNotNull( userNode );
+ assertEquals( "ACME Corporation", userNode.withArray("entities").get(0).get("employer").asText());
}
}
[04/13] git commit: Merge branch 'two-dot-o' of
github.com:apache/incubator-usergrid into 2.0
Posted by to...@apache.org.
Merge branch 'two-dot-o' of github.com:apache/incubator-usergrid into 2.0
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/7bb9c647
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/7bb9c647
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/7bb9c647
Branch: refs/heads/two-dot-o
Commit: 7bb9c6475004e7eab36275cdf68394344208b1af
Parents: 76b3421 1e7da5c
Author: amuramoto <am...@apigee.com>
Authored: Wed Oct 1 15:33:20 2014 -0700
Committer: amuramoto <am...@apigee.com>
Committed: Wed Oct 1 15:33:20 2014 -0700
----------------------------------------------------------------------
.../mq/cassandra/io/AbstractSearch.java | 60 +-
.../mq/cassandra/io/ConsumerTransaction.java | 2 +-
.../collection/EntityCollectionManager.java | 12 +-
.../collection/event/EntityDeleted.java | 25 +
.../collection/event/EntityVersionCreated.java | 25 +
.../collection/event/EntityVersionDeleted.java | 29 +
.../collection/guice/CollectionModule.java | 18 +-
.../guice/CollectionTaskExecutor.java | 35 +
.../impl/EntityCollectionManagerImpl.java | 14 +
.../impl/EntityVersionCleanupTask.java | 198 ++++++
.../serialization/SerializationFig.java | 32 +-
.../serialization/impl/LogEntryIterator.java | 114 +++
.../EntityCollectionManagerStressTest.java | 2 +
.../impl/EntityVersionCleanupTaskTest.java | 690 +++++++++++++++++++
.../impl/LogEntryIteratorTest.java | 131 ++++
.../collection/util/LogEntryMock.java | 152 ++++
.../core/astyanax/AstyanaxKeyspaceProvider.java | 2 +
.../persistence/core/guice/CommonModule.java | 2 +
.../core/task/NamedTaskExecutorImpl.java | 167 +++++
.../usergrid/persistence/core/task/Task.java | 31 +
.../persistence/core/task/TaskExecutor.java | 23 +
.../core/task/NamedTaskExecutorImplTest.java | 227 ++++++
.../usergrid/persistence/graph/GraphFig.java | 2 +
.../persistence/graph/event/EdgeDeleted.java | 8 +
.../persistence/graph/guice/GraphModule.java | 16 +
.../graph/guice/GraphTaskExecutor.java | 33 +
.../impl/shard/impl/NodeShardCacheImpl.java | 5 -
.../shard/impl/ShardGroupCompactionImpl.java | 175 +++--
.../graph/GraphManagerStressTest.java | 1 +
.../impl/shard/ShardGroupCompactionTest.java | 5 +-
.../notifications/ApplicationQueueManager.java | 42 +-
.../notifications/SingleQueueTaskManager.java | 177 -----
.../services/notifications/TaskManager.java | 180 +++++
.../services/notifications/TaskTracker.java | 4 +-
.../apns/NotificationsServiceIT.java | 13 +-
.../gcm/NotificationsServiceIT.java | 8 +-
36 files changed, 2355 insertions(+), 305 deletions(-)
----------------------------------------------------------------------
[07/13] git commit: First pass of changing cp and rm manager to use
batches
Posted by to...@apache.org.
First pass of changing cp and rm manager to use batches
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/bba08ddc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/bba08ddc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/bba08ddc
Branch: refs/heads/two-dot-o
Commit: bba08ddc1ed27b719ecaf46c63eb3f20d4bef8f5
Parents: fba71c2
Author: Todd Nine <to...@apache.org>
Authored: Wed Oct 1 22:47:06 2014 -0600
Committer: Todd Nine <to...@apache.org>
Committed: Wed Oct 1 22:47:06 2014 -0600
----------------------------------------------------------------------
.../CpEntityIndexDeleteListener.java | 8 +-
.../corepersistence/CpEntityManager.java | 40 ++++++---
.../corepersistence/CpEntityManagerFactory.java | 33 ++++---
.../corepersistence/CpManagerCache.java | 12 +--
.../corepersistence/CpRelationManager.java | 93 +++++++++++++-------
.../CpEntityIndexDeleteListenerTest.java | 12 ++-
.../core/scope/ApplicationScopeImpl.java | 6 +-
.../serialization/EdgeSerializationTest.java | 3 +
.../index/impl/EsEntityIndexBatchImpl.java | 42 +++++++--
.../index/impl/EsEntityIndexImpl.java | 2 +-
.../index/utils/IndexValidationUtils.java | 10 +++
11 files changed, 177 insertions(+), 84 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bba08ddc/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityIndexDeleteListener.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityIndexDeleteListener.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityIndexDeleteListener.java
index f8b28ac..cf3f4a6 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityIndexDeleteListener.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityIndexDeleteListener.java
@@ -59,13 +59,13 @@ public class CpEntityIndexDeleteListener {
public Observable<EntityVersion> receive(final MvccEntityDeleteEvent event) {
- CollectionScope collectionScope = event.getCollectionScope();
- IndexScope indexScope = new IndexScopeImpl(collectionScope.getApplication(), collectionScope.getOwner(), collectionScope.getName());
+ final CollectionScope collectionScope = event.getCollectionScope();
+ final IndexScope indexScope = new IndexScopeImpl(collectionScope.getApplication(), collectionScope.getOwner(), collectionScope.getName());
final EntityIndex entityIndex = entityIndexFactory.createEntityIndex(indexScope);
return Observable.create(new ObservableIterator<CandidateResult>("deleteEsIndexVersions") {
@Override
protected Iterator<CandidateResult> getIterator() {
- CandidateResults results = entityIndex.getEntityVersions(event.getEntity().getId());
+ CandidateResults results = entityIndex.getEntityVersions(indexScope, event.getEntity().getId());
return results.iterator();
}
}).subscribeOn(Schedulers.io())
@@ -78,7 +78,7 @@ public class CpEntityIndexDeleteListener {
//filter find entities <= current version
if (entity.getVersion().timestamp() <= event.getVersion().timestamp()) {
versions.add(entity);
- entityIndex.deindex(entity.getId(), entity.getVersion());
+ entityIndex.createBatch().deindex(indexScope, entity.getId(), entity.getVersion());
}
}
return Observable.from(versions);
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bba08ddc/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index f85ba30..a5475e7 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -74,6 +74,7 @@ import org.apache.usergrid.persistence.exceptions.EntityNotFoundException;
import org.apache.usergrid.persistence.exceptions.RequiredPropertyNotFoundException;
import org.apache.usergrid.persistence.exceptions.UnexpectedEntityTypeException;
import org.apache.usergrid.persistence.index.EntityIndex;
+import org.apache.usergrid.persistence.index.EntityIndexBatch;
import org.apache.usergrid.persistence.index.IndexScope;
import org.apache.usergrid.persistence.index.impl.IndexScopeImpl;
import org.apache.usergrid.persistence.index.query.CounterResolution;
@@ -561,6 +562,12 @@ public class CpEntityManager implements EntityManager {
logger.debug( "Deleting indexes of all {} collections owning the entity",
owners.keySet().size() );
+ final EntityIndex ei = managerCache.getEntityIndex( appScope );
+
+ final EntityIndexBatch batch = ei.createBatch();
+
+
+
for ( String ownerType : owners.keySet() ) {
Map<UUID, Set<String>> collectionsByUuid = owners.get( ownerType );
@@ -573,27 +580,30 @@ public class CpEntityManager implements EntityManager {
new SimpleId( uuid, ownerType ),
CpEntityManager.getCollectionScopeNameFromCollectionName(coll) );
- EntityIndex ei = managerCache.getEntityIndex( indexScope );
- ei.deindex( entity );
+ batch.index( indexScope, entity );
}
}
}
+
+
// deindex from default index scope
IndexScope defaultIndexScope = new IndexScopeImpl(
appScope.getApplication(),
appScope.getApplication(),
getCollectionScopeNameFromEntityType( entityRef.getType() ) );
- EntityIndex entityIndex = managerCache.getEntityIndex( defaultIndexScope );
- entityIndex.deindex( entity );
+
+ batch.deindex(defaultIndexScope, entity );
IndexScope allTypesIndexScope = new IndexScopeImpl(
appScope.getApplication(),
appScope.getApplication(),
ALL_TYPES);
- EntityIndex aei = managerCache.getEntityIndex( allTypesIndexScope );
- aei.deindex( entity );
+
+ batch.deindex( allTypesIndexScope, entity );
+
+ batch.execute();
decrementEntityCollection( Schema.defaultCollectionName( entityId.getType() ) );
@@ -980,7 +990,7 @@ public class CpEntityManager implements EntityManager {
getCollectionScopeNameFromEntityType( entityRef.getType()) );
EntityCollectionManager ecm = managerCache.getEntityCollectionManager( collectionScope );
- EntityIndex ei = managerCache.getEntityIndex( defaultIndexScope );
+ EntityIndex ei = managerCache.getEntityIndex( appScope );
Id entityId = new SimpleId( entityRef.getUuid(), entityRef.getType() );
@@ -1002,7 +1012,8 @@ public class CpEntityManager implements EntityManager {
logger.debug("Wrote {}:{} version {}", new Object[] {
cpEntity.getId().getType(), cpEntity.getId().getUuid(), cpEntity.getVersion() });
- ei.index( cpEntity );
+
+ ei.createBatch().index(defaultIndexScope, cpEntity ).execute();
// update in all containing collections and connection indexes
CpRelationManager rm = (CpRelationManager)getRelationManager( entityRef );
@@ -2530,12 +2541,13 @@ public class CpEntityManager implements EntityManager {
}
// Index CP entity into default collection scope
- IndexScope defaultIndexScope = new IndexScopeImpl(
- appScope.getApplication(),
- appScope.getApplication(),
- CpEntityManager.getCollectionScopeNameFromEntityType( entity.getType() ) );
- EntityIndex ei = managerCache.getEntityIndex( defaultIndexScope );
- ei.index( cpEntity );
+// IndexScope defaultIndexScope = new IndexScopeImpl(
+// appScope.getApplication(),
+// appScope.getApplication(),
+// CpEntityManager.getCollectionScopeNameFromEntityType( entity.getType() ) );
+// EntityIndex ei = managerCache.getEntityIndex( appScope );
+//
+// ei.createBatch().index( defaultIndexScope, cpEntity ).execute();
// reflect changes in the legacy Entity
entity.setUuid( cpEntity.getId().getUuid() );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bba08ddc/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index 3e9a6a9..090a4d6 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -50,6 +50,7 @@ import org.apache.usergrid.persistence.exceptions.ApplicationAlreadyExistsExcept
import org.apache.usergrid.persistence.exceptions.DuplicateUniquePropertyExistsException;
import org.apache.usergrid.persistence.graph.GraphManagerFactory;
import org.apache.usergrid.persistence.index.EntityIndex;
+import org.apache.usergrid.persistence.index.EntityIndexBatch;
import org.apache.usergrid.persistence.index.EntityIndexFactory;
import org.apache.usergrid.persistence.index.IndexScope;
import org.apache.usergrid.persistence.index.query.CandidateResult;
@@ -263,8 +264,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
.getEntityIndex( SYSTEM_ORGS_INDEX_SCOPE );
orgInfoEntity = ecm.write( orgInfoEntity ).toBlockingObservable().last();
- eci.index( orgInfoEntity );
- eci.refresh();
+ eci.createBatch().index(SYSTEM_ORGS_INDEX_SCOPE, orgInfoEntity ).executeAndRefresh();
}
if ( properties == null ) {
@@ -288,7 +288,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
.getEntityIndex( SYSTEM_APPS_INDEX_SCOPE );
appInfoEntity = ecm.write( appInfoEntity ).toBlockingObservable().last();
- eci.index( appInfoEntity );
+ eci.createBatch().index(SYSTEM_APPS_INDEX_SCOPE, appInfoEntity ).executeAndRefresh();
eci.refresh();
}
@@ -325,7 +325,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
Query q = Query.fromQL(PROPERTY_NAME + " = '" + name + "'");
EntityIndex ei = getManagerCache().getEntityIndex( SYSTEM_ORGS_INDEX_SCOPE );
- CandidateResults results = ei.search( q );
+ CandidateResults results = ei.search(SYSTEM_ORGS_INDEX_SCOPE, q );
if ( results.isEmpty() ) {
return null;
@@ -342,7 +342,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
EntityIndex ei = getManagerCache().getEntityIndex( SYSTEM_APPS_INDEX_SCOPE );
- CandidateResults results = ei.search( q );
+ CandidateResults results = ei.search(SYSTEM_APPS_INDEX_SCOPE, q );
if ( results.isEmpty() ) {
return null;
@@ -371,7 +371,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
Query q = Query.fromQL("select *");
q.setCursor( cursor );
- CandidateResults results = ei.search( q );
+ CandidateResults results = ei.search(SYSTEM_APPS_INDEX_SCOPE, q );
cursor = results.getCursor();
Iterator<CandidateResult> iter = results.iterator();
@@ -417,7 +417,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
Query q = Query.fromQL("select *");
- CandidateResults results = ei.search( q );
+ CandidateResults results = ei.search(SYSTEM_PROPS_INDEX_SCOPE, q );
if ( results.isEmpty() ) {
return new HashMap<String,String>();
@@ -447,7 +447,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
.getEntityIndex( SYSTEM_PROPS_INDEX_SCOPE );
Query q = Query.fromQL("select *");
- CandidateResults results = ei.search( q );
+ CandidateResults results = ei.search(SYSTEM_PROPS_INDEX_SCOPE, q );
Entity propsEntity;
if ( !results.isEmpty() ) {
propsEntity = em.load( results.iterator().next().getId()).toBlockingObservable().last();
@@ -464,7 +464,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
}
propsEntity = em.write( propsEntity ).toBlockingObservable().last();
- ei.index( propsEntity );
+ ei.createBatch().index( SYSTEM_PROPS_INDEX_SCOPE, propsEntity );
return true;
}
@@ -485,7 +485,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
EntityIndex ei = getManagerCache().getEntityIndex( SYSTEM_PROPS_INDEX_SCOPE );
Query q = Query.fromQL("select *");
- CandidateResults results = ei.search( q );
+ CandidateResults results = ei.search(SYSTEM_PROPS_INDEX_SCOPE, q );
Entity propsEntity = em.load(
results.iterator().next().getId() ).toBlockingObservable().last();
@@ -501,7 +501,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
propsEntity.removeField( name );
propsEntity = em.write( propsEntity ).toBlockingObservable().last();
- ei.index( propsEntity );
+ ei.createBatch().index( SYSTEM_PROPS_INDEX_SCOPE, propsEntity );
return true;
}
@@ -616,9 +616,10 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
EntityIndex ei = managerCache.getEntityIndex( is );
Query q = Query.fromQL("select *");
- CandidateResults results = ei.search( q );
+ CandidateResults results = ei.search(is, q );
- Map<String, UUID> appMap = new HashMap<String, UUID>();
+ int count = 0;
+ final EntityIndexBatch batch = ei.createBatch();
Iterator<CandidateResult> iter = results.iterator();
while (iter.hasNext()) {
@@ -643,7 +644,11 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
}
);
- ei.index(entity);
+ batch.index(is, entity);
+
+
+
+ count++;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bba08ddc/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpManagerCache.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpManagerCache.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpManagerCache.java
index 2f8bd3f..0e7c084 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpManagerCache.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpManagerCache.java
@@ -36,8 +36,8 @@ class CpManagerCache {
private final LRUCache2<CollectionScope, EntityCollectionManager> ecmCache
= new LRUCache2<CollectionScope, EntityCollectionManager>(50, 1 * 60 * 60 * 1000);
- private final LRUCache2<IndexScope, EntityIndex> eiCache
- = new LRUCache2<IndexScope, EntityIndex>(50, 1 * 60 * 60 * 1000);
+ private final LRUCache2<ApplicationScope, EntityIndex> eiCache
+ = new LRUCache2<>(50, 1 * 60 * 60 * 1000);
private final LRUCache2<ApplicationScope, GraphManager> gmCache
= new LRUCache2<ApplicationScope, GraphManager>(50, 1 * 60 * 60 * 1000);
@@ -61,13 +61,13 @@ class CpManagerCache {
return ecm;
}
- public EntityIndex getEntityIndex(IndexScope indexScope) {
+ public EntityIndex getEntityIndex(ApplicationScope applicationScope) {
- EntityIndex ei = eiCache.get(indexScope);
+ EntityIndex ei = eiCache.get(applicationScope);
if (ei == null) {
- ei = eif.createEntityIndex(indexScope);
- eiCache.put(indexScope, ei);
+ ei = eif.createEntityIndex(applicationScope);
+ eiCache.put(applicationScope, ei);
}
return ei;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bba08ddc/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index 3486fbc..260e429 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -28,6 +28,7 @@ import java.util.Map;
import java.util.Set;
import java.util.UUID;
+import org.apache.usergrid.persistence.index.EntityIndexBatch;
import org.apache.usergrid.utils.UUIDUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -425,6 +426,11 @@ public class CpRelationManager implements RelationManager {
// loop through all types of edge to target
int count = 0;
+
+ final EntityIndex ei = managerCache.getEntityIndex( applicationScope );
+
+ final EntityIndexBatch entityIndexBatch = ei.createBatch();
+
while ( edgeTypesToTarget.hasNext() ) {
// get all edges of the type
@@ -460,10 +466,9 @@ public class CpRelationManager implements RelationManager {
applicationScope.getApplication(),
new SimpleId(sourceEntity.getUuid(), sourceEntity.getType()),
CpEntityManager.getConnectionScopeName( cpHeadEntity.getId().getType(), connName ));
- }
-
- EntityIndex ei = managerCache.getEntityIndex(indexScope);
- ei.index(cpEntity);
+ }
+
+ entityIndexBatch.index(indexScope, cpEntity);
// reindex the entity in the source entity's all-types index
@@ -471,8 +476,8 @@ public class CpRelationManager implements RelationManager {
applicationScope.getApplication(),
new SimpleId(sourceEntity.getUuid(), sourceEntity.getType()),
ALL_TYPES);
- ei = managerCache.getEntityIndex(indexScope);
- ei.index(cpEntity);
+
+ entityIndexBatch.index(indexScope, cpEntity);
count++;
}
@@ -659,6 +664,7 @@ public class CpRelationManager implements RelationManager {
applicationScope.getApplication(),
applicationScope.getApplication(),
CpEntityManager.getCollectionScopeNameFromEntityType( itemRef.getType()));
+
EntityCollectionManager memberMgr = managerCache.getEntityCollectionManager(memberScope);
org.apache.usergrid.persistence.model.entity.Entity memberEntity = memberMgr.load(
@@ -700,29 +706,36 @@ public class CpRelationManager implements RelationManager {
GraphManager gm = managerCache.getGraphManager(applicationScope);
gm.writeEdge(edge).toBlockingObservable().last();
+ final EntityIndex index = managerCache.getEntityIndex( applicationScope );
+
+ final EntityIndexBatch batch = index.createBatch();
+
+
// index member into entity collection | type scope
IndexScope collectionIndexScope = new IndexScopeImpl(
applicationScope.getApplication(),
cpHeadEntity.getId(),
CpEntityManager.getCollectionScopeNameFromCollectionName( collName ));
- EntityIndex collectionIndex = managerCache.getEntityIndex(collectionIndexScope);
- collectionIndex.index( memberEntity );
+
+ batch.index(collectionIndexScope, memberEntity );
// index member into entity | all-types scope
IndexScope entityAllTypesScope = new IndexScopeImpl(
applicationScope.getApplication(),
cpHeadEntity.getId(),
ALL_TYPES);
- EntityIndex entityAllCollectionIndex = managerCache.getEntityIndex(entityAllTypesScope);
- entityAllCollectionIndex.index( memberEntity );
+
+ batch.index(entityAllTypesScope, memberEntity );
// index member into application | all-types scope
IndexScope appAllTypesScope = new IndexScopeImpl(
applicationScope.getApplication(),
applicationScope.getApplication(),
ALL_TYPES);
- EntityIndex allCollectionIndex = managerCache.getEntityIndex(appAllTypesScope);
- allCollectionIndex.index( memberEntity );
+
+ batch.index( appAllTypesScope, memberEntity );
+
+ batch.execute();
logger.debug("Added entity {}:{} to collection {}", new String[] {
itemRef.getUuid().toString(), itemRef.getType(), collName });
@@ -844,13 +857,16 @@ public class CpRelationManager implements RelationManager {
org.apache.usergrid.persistence.model.entity.Entity memberEntity = memberMgr.load(
new SimpleId( itemRef.getUuid(), itemRef.getType() )).toBlockingObservable().last();
+ final EntityIndex ei = managerCache.getEntityIndex(applicationScope);
+ final EntityIndexBatch batch = ei.createBatch();
+
// remove item from collection index
IndexScope indexScope = new IndexScopeImpl(
applicationScope.getApplication(),
cpHeadEntity.getId(),
CpEntityManager.getCollectionScopeNameFromCollectionName( collName ));
- EntityIndex ei = managerCache.getEntityIndex(indexScope);
- ei.deindex( memberEntity );
+
+ batch.deindex(indexScope, memberEntity );
// remove collection from item index
IndexScope itemScope = new IndexScopeImpl(
@@ -858,8 +874,11 @@ public class CpRelationManager implements RelationManager {
memberEntity.getId(),
CpEntityManager.getCollectionScopeNameFromCollectionName(
Schema.defaultCollectionName( cpHeadEntity.getId().getType() )));
- ei = managerCache.getEntityIndex(itemScope);
- ei.deindex( cpHeadEntity );
+
+
+ batch.deindex(itemScope, cpHeadEntity );
+
+ batch.execute();
// remove edge from collection to item
GraphManager gm = managerCache.getGraphManager(applicationScope);
@@ -958,7 +977,8 @@ public class CpRelationManager implements RelationManager {
applicationScope.getApplication(),
cpHeadEntity.getId(),
CpEntityManager.getCollectionScopeNameFromCollectionName( collName ));
- EntityIndex ei = managerCache.getEntityIndex(indexScope);
+
+ EntityIndex ei = managerCache.getEntityIndex(applicationScope);
logger.debug("Searching scope {}:{}:{}",
new String[] {
@@ -969,7 +989,7 @@ public class CpRelationManager implements RelationManager {
query.setEntityType( collection.getType() );
query = adjustQuery( query );
- CandidateResults crs = ei.search( query );
+ CandidateResults crs = ei.search(indexScope, query );
return buildResults( query, crs, collName );
@@ -1088,21 +1108,25 @@ public class CpRelationManager implements RelationManager {
GraphManager gm = managerCache.getGraphManager(applicationScope);
gm.writeEdge(edge).toBlockingObservable().last();
+ final EntityIndex ei = managerCache.getEntityIndex(applicationScope);
+ final EntityIndexBatch batch = ei.createBatch();
+
// Index the new connection in app|source|type context
IndexScope indexScope = new IndexScopeImpl(
applicationScope.getApplication(),
cpHeadEntity.getId(),
CpEntityManager.getConnectionScopeName( connectedEntityRef.getType(), connectionType ));
- EntityIndex ei = managerCache.getEntityIndex(indexScope);
- ei.index( targetEntity );
+ batch.index( indexScope, targetEntity );
// Index the new connection in app|scope|all-types context
IndexScope allTypesIndexScope = new IndexScopeImpl(
applicationScope.getApplication(),
cpHeadEntity.getId(),
ALL_TYPES);
- EntityIndex aei = managerCache.getEntityIndex(allTypesIndexScope);
- aei.index( targetEntity );
+
+ batch.index( allTypesIndexScope, targetEntity );
+
+ batch.execute();
Keyspace ko = cass.getApplicationKeyspace( applicationId );
Mutator<ByteBuffer> m = createMutator( ko, be );
@@ -1307,21 +1331,23 @@ public class CpRelationManager implements RelationManager {
GraphManager gm = managerCache.getGraphManager(applicationScope);
gm.deleteEdge(edge).toBlockingObservable().last();
+ final EntityIndex ei = managerCache.getEntityIndex( applicationScope ) ;
+ final EntityIndexBatch batch = ei.createBatch();
+
// Deindex the connection in app|source|type context
IndexScope indexScope = new IndexScopeImpl(
applicationScope.getApplication(),
new SimpleId( connectingEntityRef.getUuid(), connectingEntityRef.getType() ),
CpEntityManager.getConnectionScopeName( targetEntity.getId().getType(), connectionType ));
- EntityIndex ei = managerCache.getEntityIndex( indexScope );
- ei.deindex( targetEntity );
+ batch.deindex( indexScope , targetEntity );
// Deindex the connection in app|source|type context
IndexScope allTypesIndexScope = new IndexScopeImpl(
applicationScope.getApplication(),
new SimpleId( connectingEntityRef.getUuid(), connectingEntityRef.getType() ),
ALL_TYPES);
- EntityIndex aei = managerCache.getEntityIndex(allTypesIndexScope);
- aei.deindex( targetEntity );
+
+ batch.deindex( allTypesIndexScope, targetEntity );
}
@@ -1377,7 +1403,9 @@ public class CpRelationManager implements RelationManager {
applicationScope.getApplication(),
cpHeadEntity.getId(),
scopeName);
- EntityIndex ei = managerCache.getEntityIndex(indexScope);
+
+ final EntityIndex ei = managerCache.getEntityIndex(applicationScope);
+
logger.debug("Searching connected entities from scope {}:{}:{}", new String[] {
indexScope.getApplication().toString(),
@@ -1385,7 +1413,7 @@ public class CpRelationManager implements RelationManager {
indexScope.getName()});
query = adjustQuery( query );
- CandidateResults crs = ei.search( query );
+ CandidateResults crs = ei.search( indexScope, query );
raw = buildResults( query , crs, query.getConnectionType() );
}
@@ -1480,7 +1508,8 @@ public class CpRelationManager implements RelationManager {
applicationScope.getApplication(),
cpHeadEntity.getId(),
ALL_TYPES);
- EntityIndex ei = managerCache.getEntityIndex(indexScope);
+
+ EntityIndex ei = managerCache.getEntityIndex(applicationScope);
logger.debug("Searching connections from the all-types scope {}:{}:{}", new String[] {
indexScope.getApplication().toString(),
@@ -1488,7 +1517,7 @@ public class CpRelationManager implements RelationManager {
indexScope.getName()});
query = adjustQuery( query );
- CandidateResults crs = ei.search( query );
+ CandidateResults crs = ei.search(indexScope, query );
return buildConnectionResults(query , crs, query.getConnectionType() );
}
@@ -1498,7 +1527,7 @@ public class CpRelationManager implements RelationManager {
cpHeadEntity.getId(),
CpEntityManager.getConnectionScopeName(
query.getEntityType(), query.getConnectionType() ));
- EntityIndex ei = managerCache.getEntityIndex(indexScope);
+ EntityIndex ei = managerCache.getEntityIndex(applicationScope);
logger.debug("Searching connections from the '{}' scope {}:{}:{}", new String[] {
indexScope.getApplication().toString(),
@@ -1506,7 +1535,7 @@ public class CpRelationManager implements RelationManager {
indexScope.getName()});
query = adjustQuery( query );
- CandidateResults crs = ei.search( query );
+ CandidateResults crs = ei.search( indexScope, query );
return buildConnectionResults(query , crs, query.getConnectionType() );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bba08ddc/stack/core/src/test/java/org/apache/usergrid/corepersistence/CpEntityIndexDeleteListenerTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/CpEntityIndexDeleteListenerTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/CpEntityIndexDeleteListenerTest.java
index 6ee62c6..d59432b 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/CpEntityIndexDeleteListenerTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/CpEntityIndexDeleteListenerTest.java
@@ -35,6 +35,7 @@ import org.apache.usergrid.persistence.collection.mvcc.entity.impl.MvccEntityImp
import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
import org.apache.usergrid.persistence.core.entity.EntityVersion;
import org.apache.usergrid.persistence.index.EntityIndex;
+import org.apache.usergrid.persistence.index.EntityIndexBatch;
import org.apache.usergrid.persistence.index.EntityIndexFactory;
import org.apache.usergrid.persistence.index.IndexScope;
import org.apache.usergrid.persistence.index.query.CandidateResult;
@@ -82,6 +83,10 @@ public class CpEntityIndexDeleteListenerTest {
when(scope.getApplication()).thenReturn(entityId);
when(eif.createEntityIndex(any(IndexScope.class))).thenReturn(entityIndex);
+ final EntityIndexBatch batch = mock(EntityIndexBatch.class);
+
+ when(entityIndex.createBatch()).thenReturn( batch );
+
CandidateResults results = mock(CandidateResults.class);
List<CandidateResult> resultsList = new ArrayList<>();
resultsList.add(entity);
@@ -90,7 +95,7 @@ public class CpEntityIndexDeleteListenerTest {
when(results.iterator()).thenReturn(entities);
when(serializationFig.getBufferSize()).thenReturn(10);
when(serializationFig.getHistorySize()).thenReturn(20);
- when(entityIndex.getEntityVersions(entityId)).thenReturn(results);
+ when(entityIndex.getEntityVersions(any(IndexScope.class), entityId)).thenReturn(results);
MvccEntity mvccEntity = new MvccEntityImpl(entityId,uuid, MvccEntity.Status.COMPLETE,mock(Entity.class));
@@ -98,6 +103,9 @@ public class CpEntityIndexDeleteListenerTest {
Observable<EntityVersion> o = esEntityIndexDeleteListener.receive(event);
EntityVersion testEntity = o.toBlocking().last();
assertEquals(testEntity.getId(),mvccEntity.getId());
- verify(entityIndex).deindex(entity.getId(),entity.getVersion());
+
+ verify(entityIndex).createBatch();
+
+ verify(batch).deindex(any(IndexScope.class), entity.getId(),entity.getVersion());
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bba08ddc/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/scope/ApplicationScopeImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/scope/ApplicationScopeImpl.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/scope/ApplicationScopeImpl.java
index 4e067c2..e8dbb02 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/scope/ApplicationScopeImpl.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/scope/ApplicationScopeImpl.java
@@ -48,13 +48,13 @@ public class ApplicationScopeImpl implements ApplicationScope {
if ( this == o ) {
return true;
}
- if ( !( o instanceof ApplicationScopeImpl ) ) {
+ if ( !( o instanceof ApplicationScope ) ) {
return false;
}
- final ApplicationScopeImpl that = ( ApplicationScopeImpl ) o;
+ final ApplicationScope that = ( ApplicationScope ) o;
- if ( !application.equals( that.application ) ) {
+ if ( !application.equals( that.getApplication() ) ) {
return false;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bba08ddc/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/EdgeSerializationTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/EdgeSerializationTest.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/EdgeSerializationTest.java
index 9c29d56..57391de 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/EdgeSerializationTest.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/EdgeSerializationTest.java
@@ -27,6 +27,7 @@ import java.util.UUID;
import org.junit.Before;
import org.junit.ClassRule;
+import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.slf4j.Logger;
@@ -656,6 +657,7 @@ public abstract class EdgeSerializationTest {
* Test paging by resuming the search from the edge
*/
@Test
+ @Ignore("Kills embedded cassandra")
public void pageIteration() throws ConnectionException {
int size = graphFig.getScanPageSize() * 2;
@@ -695,6 +697,7 @@ public abstract class EdgeSerializationTest {
* edge types
*/
@Test
+ @Ignore("Kills embedded cassandra")
public void testIteratorPaging() throws ConnectionException {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bba08ddc/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
index d9857f2..93f0e41 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
@@ -42,6 +42,7 @@ import org.apache.usergrid.persistence.index.EntityIndexBatch;
import org.apache.usergrid.persistence.index.IndexFig;
import org.apache.usergrid.persistence.index.IndexScope;
import org.apache.usergrid.persistence.index.query.CandidateResult;
+import org.apache.usergrid.persistence.index.utils.IndexValidationUtils;
import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.field.ArrayField;
@@ -91,15 +92,20 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
private BulkRequestBuilder bulkRequest;
+ private final int autoFlushSize;
- public EsEntityIndexBatchImpl( final ApplicationScope applicationScope,
- final Client client, final IndexFig config, final Set<String> knownTypes ) {
+ private int count;
+
+
+ public EsEntityIndexBatchImpl( final ApplicationScope applicationScope, final Client client, final IndexFig config,
+ final Set<String> knownTypes, final int autoFlushSize ) {
this.applicationScope = applicationScope;
this.client = client;
this.knownTypes = knownTypes;
this.indexName = createIndexName( config.getIndexPrefix(), applicationScope );
this.refresh = config.isForcedRefresh();
+ this.autoFlushSize = autoFlushSize;
initBatch();
}
@@ -107,6 +113,9 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
@Override
public EntityIndexBatch index( final IndexScope indexScope, final Entity entity ) {
+
+ IndexValidationUtils.validateScopeMatch( indexScope, applicationScope );
+
final String indexType = createCollectionScopeTypeName( indexScope );
if ( log.isDebugEnabled() ) {
@@ -118,7 +127,7 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
ValidationUtils.verifyEntityWrite( entity );
- initType(indexScope, indexType );
+ initType( indexScope, indexType );
Map<String, Object> entityAsMap = entityToMap( entity );
@@ -135,12 +144,16 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
bulkRequest.add( client.prepareIndex( indexName, indexType, indexId ).setSource( entityAsMap ) );
+ maybeFlush();
+
return this;
}
@Override
- public EntityIndexBatch deindex(final IndexScope indexScope, final Id id, final UUID version ) {
+ public EntityIndexBatch deindex( final IndexScope indexScope, final Id id, final UUID version ) {
+
+ IndexValidationUtils.validateScopeMatch( indexScope, applicationScope );
final String indexType = createCollectionScopeTypeName( indexScope );
@@ -158,6 +171,8 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
log.debug( "Deindexed Entity with index id " + indexId );
+ maybeFlush();
+
return this;
}
@@ -165,14 +180,14 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
@Override
public EntityIndexBatch deindex( final IndexScope indexScope, final Entity entity ) {
- return deindex(indexScope, entity.getId(), entity.getVersion() );
+ return deindex( indexScope, entity.getId(), entity.getVersion() );
}
@Override
public EntityIndexBatch deindex( final IndexScope indexScope, final CandidateResult entity ) {
- return deindex(indexScope, entity.getId(), entity.getVersion() );
+ return deindex( indexScope, entity.getId(), entity.getVersion() );
}
@@ -184,7 +199,6 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
/**
* Execute the request, check for errors, then re-init the batch for future use
- * @param request
*/
private void execute( final BulkRequestBuilder request ) {
final BulkResponse response = request.execute().actionGet();
@@ -199,7 +213,17 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
@Override
public void executeAndRefresh() {
- execute(bulkRequest.setRefresh( true ) );
+ execute( bulkRequest.setRefresh( true ) );
+ }
+
+
+ private void maybeFlush() {
+ count++;
+
+ if ( count % autoFlushSize == 0 ) {
+ execute();
+ count = 0;
+ }
}
@@ -217,6 +241,8 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
try {
XContentBuilder mxcb = EsEntityIndexImpl.createDoubleStringIndexMapping( jsonBuilder(), typeName );
+
+ //TODO Dave can this be collapsed into the build as well?
admin.indices().preparePutMapping( indexName ).setType( typeName ).setSource( mxcb ).execute().actionGet();
admin.indices().prepareGetMappings( indexName ).addTypes( typeName ).execute().actionGet();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bba08ddc/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 077036a..56316d8 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -134,7 +134,7 @@ public class EsEntityIndexImpl implements EntityIndex {
@Override
public EntityIndexBatch createBatch() {
- return new EsEntityIndexBatchImpl( applicationScope, client, config, knownTypes );
+ return new EsEntityIndexBatchImpl( applicationScope, client, config, knownTypes, 1000 );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bba08ddc/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/utils/IndexValidationUtils.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/utils/IndexValidationUtils.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/utils/IndexValidationUtils.java
index 899e7b0..d6080de 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/utils/IndexValidationUtils.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/utils/IndexValidationUtils.java
@@ -19,6 +19,7 @@
package org.apache.usergrid.persistence.index.utils;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.index.IndexScope;
import com.google.common.base.Preconditions;
@@ -50,4 +51,13 @@ public class IndexValidationUtils {
}
+ /**
+ * Validate the scope in the index matches the application scope
+ * @param indexScope
+ * @param scope
+ */
+ public static void validateScopeMatch(final IndexScope indexScope,final ApplicationScope scope){
+ Preconditions.checkArgument( scope.equals( indexScope ) );
+ }
+
}
[03/13] git commit: update to partialUpdateTest
Posted by to...@apache.org.
update to partialUpdateTest
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/76b3421b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/76b3421b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/76b3421b
Branch: refs/heads/two-dot-o
Commit: 76b3421bad587c2339bd9a8772179f9586fae751
Parents: 16d497e
Author: amuramoto <am...@apigee.com>
Authored: Tue Sep 30 16:37:34 2014 -0700
Committer: amuramoto <am...@apigee.com>
Committed: Tue Sep 30 16:37:34 2014 -0700
----------------------------------------------------------------------
.../apache/usergrid/rest/PartialUpdateTest.java | 86 +++++++++-----------
1 file changed, 38 insertions(+), 48 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/76b3421b/stack/rest/src/test/java/org/apache/usergrid/rest/PartialUpdateTest.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/PartialUpdateTest.java b/stack/rest/src/test/java/org/apache/usergrid/rest/PartialUpdateTest.java
index c2f3fcb..7053160 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/PartialUpdateTest.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/PartialUpdateTest.java
@@ -23,19 +23,14 @@ import java.io.IOException;
import java.util.LinkedHashMap;
import java.util.Map;
import javax.ws.rs.core.MediaType;
-<<<<<<< HEAD
-=======
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.fail;
->>>>>>> e5bcbb23be66d12e428d817fd8b0b495793a939c
import org.junit.Rule;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static org.junit.Assert.*;
+import static org.junit.Assert.assertNotEquals;
/**
@@ -63,14 +58,7 @@ public class PartialUpdateTest extends AbstractRestIT {
put( "username", "bart" );
put( "employer", "Brawndo" );
put( "email", "bart@personal-email.example.com" );
-<<<<<<< HEAD
put( "location", geolocation);
-=======
- put( "location", new LinkedHashMap<String, Object>() {{
- put("latitude", "37.3638875323994");
- put("longitude", "-122.12334411518498");
- }} );
->>>>>>> e5bcbb23be66d12e428d817fd8b0b495793a939c
}};
JsonNode userNode = mapper.readTree(
@@ -85,14 +73,13 @@ public class PartialUpdateTest extends AbstractRestIT {
assertNotNull( uuid );
refreshIndex( "test-organization", "test-app" );
+ Map<String, Object> updateProperties = new LinkedHashMap<String, Object>();
// update user bart passing only an update to a property
- for(int i=1; i<20; i++) {
+ for(int i=1; i<10; i++) {
geolocation.put("latitude", latitude += 0.00001);
geolocation.put("longitude", longitude += 0.00001);
- Map<String, Object> updateProperties = new LinkedHashMap<String, Object>() {{
- put("employer", "Initech");
- put("location", geolocation);
- }};
+ updateProperties.put("employer", "Initech");
+ updateProperties.put("location", geolocation);
try {
JsonNode updatedNode = mapper.readTree(
@@ -102,22 +89,23 @@ public class PartialUpdateTest extends AbstractRestIT {
.type(MediaType.APPLICATION_JSON)
.put(String.class, updateProperties));
assertNotNull(updatedNode);
- assertNotEquals(latitude, updatedNode.get("entities").get(0).get("location").get("latitude"));
- assertNotEquals(longitude, updatedNode.get("entities").get(0).get("location").get("longitude"));
+
} catch (UniformInterfaceException uie) {
fail("Update failed due to: " + uie.getResponse().getEntity(String.class));
}
- }
- refreshIndex( "test-organization", "test-app" );
-
- userNode = mapper.readTree(
- resource().path( "/test-organization/test-app/users/" + uuid )
- .queryParam( "access_token", adminAccessToken )
- .accept( MediaType.APPLICATION_JSON )
- .get( String.class ));
- assertNotNull( userNode );
- assertEquals( "Initech", userNode.withArray("entities").get(0).get("employer").asText());
+ refreshIndex("test-organization", "test-app");
+
+ userNode = mapper.readTree(
+ resource().path("/test-organization/test-app/users/" + uuid)
+ .queryParam("access_token", adminAccessToken)
+ .accept(MediaType.APPLICATION_JSON)
+ .get(String.class));
+ assertNotNull(userNode);
+ assertEquals("Initech", userNode.withArray("entities").get(0).get("employer").asText());
+ assertNotEquals(latitude, userNode.withArray("entities").get(0).get("location").get("latitude"));
+ assertNotEquals(longitude, userNode.withArray("entities").get(0).get("location").get("longitude"));
+ }
// Update bart's employer without specifying any required fields
// (this time with username specified in URL)
@@ -126,25 +114,27 @@ public class PartialUpdateTest extends AbstractRestIT {
put( "employer", "ACME Corporation" );
}};
- try {
- mapper.readTree(
- resource().path( "/test-organization/test-app/users/bart")
- .queryParam( "access_token", adminAccessToken )
- .accept( MediaType.APPLICATION_JSON )
- .type( MediaType.APPLICATION_JSON )
- .put( String.class, updateProperties ));
+ for(int i=1; i<10; i++) {
+ try {
+ mapper.readTree(
+ resource().path("/test-organization/test-app/users/bart")
+ .queryParam("access_token", adminAccessToken)
+ .accept(MediaType.APPLICATION_JSON)
+ .type(MediaType.APPLICATION_JSON)
+ .put(String.class, updateProperties));
- } catch ( UniformInterfaceException uie ) {
- fail("Update failed due to: " + uie.getResponse().getEntity(String.class));
+ } catch (UniformInterfaceException uie) {
+ fail("Update failed due to: " + uie.getResponse().getEntity(String.class));
+ }
+ refreshIndex("test-organization", "test-app");
+
+ userNode = mapper.readTree(
+ resource().path("/test-organization/test-app/users/bart")
+ .queryParam("access_token", adminAccessToken)
+ .accept(MediaType.APPLICATION_JSON)
+ .get(String.class));
+ assertNotNull(userNode);
+ assertEquals("ACME Corporation", userNode.withArray("entities").get(0).get("employer").asText());
}
- refreshIndex( "test-organization", "test-app" );
-
- userNode = mapper.readTree(
- resource().path( "/test-organization/test-app/users/bart" )
- .queryParam( "access_token", adminAccessToken )
- .accept( MediaType.APPLICATION_JSON )
- .get( String.class ));
- assertNotNull( userNode );
- assertEquals( "ACME Corporation", userNode.withArray("entities").get(0).get("employer").asText());
}
}
[10/13] git commit: Merge remote-tracking branch 'amuramoto/2.0' into
tempmerge
Posted by to...@apache.org.
Merge remote-tracking branch 'amuramoto/2.0' into tempmerge
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/93471f78
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/93471f78
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/93471f78
Branch: refs/heads/two-dot-o
Commit: 93471f78dfb0276909833dda2015fba93b56375f
Parents: c051fda d1dc7cd
Author: Todd Nine <to...@apache.org>
Authored: Wed Oct 1 23:47:57 2014 -0600
Committer: Todd Nine <to...@apache.org>
Committed: Wed Oct 1 23:47:57 2014 -0600
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 9 +-
.../usergrid/persistence/CollectionIT.java | 3 -
.../usergrid/persistence/index/query/Query.java | 2 +-
.../index/query/tree/AndOperand.java | 2 +-
.../persistence/index/query/tree/Property.java | 2 +-
.../index/query/tree/StringLiteral.java | 2 +-
.../apache/usergrid/rest/PartialUpdateTest.java | 120 ++++++++++---------
7 files changed, 75 insertions(+), 65 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/93471f78/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
[12/13] git commit: Cleaned up IndexScope so that is no longer has an
impedance mismatch with the batch API.
Posted by to...@apache.org.
Cleaned up IndexScope so that is no longer has an impedance mismatch with the batch API.
Refactored edge naming for CP into it's own utility class
Moved categorization of edge types (collection|connection) into prefixes. This will also type filtering in Cassandra when seeking edge types to populate meta data.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/e0055151
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/e0055151
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/e0055151
Branch: refs/heads/two-dot-o
Commit: e00551511413a0cd9a97621842294d0add331eb3
Parents: 0e18f57
Author: Todd Nine <to...@apache.org>
Authored: Thu Oct 2 01:23:51 2014 -0600
Committer: Todd Nine <to...@apache.org>
Committed: Thu Oct 2 01:23:51 2014 -0600
----------------------------------------------------------------------
.../CpEntityIndexDeleteListener.java | 5 +-
.../corepersistence/CpEntityManager.java | 22 ++------
.../corepersistence/CpEntityManagerFactory.java | 21 +++-----
.../usergrid/corepersistence/CpNamingUtils.java | 50 ++++++++---------
.../corepersistence/CpRelationManager.java | 35 ++++--------
.../CpEntityIndexDeleteListenerTest.java | 4 +-
.../PerformanceEntityRebuildIndexTest.java | 7 ++-
.../collection/impl/CollectionScopeImpl.java | 6 +--
.../usergrid/persistence/index/IndexScope.java | 2 +-
.../index/impl/EsEntityIndexBatchImpl.java | 27 +++++++---
.../persistence/index/impl/IndexScopeImpl.java | 8 +--
.../persistence/index/impl/IndexingUtils.java | 17 +++---
.../index/utils/IndexValidationUtils.java | 9 ----
.../index/impl/CorePerformanceIT.java | 56 ++++++++++++--------
.../impl/EntityConnectionIndexImplTest.java | 7 ++-
.../persistence/index/impl/EntityIndexTest.java | 34 ++++++++----
16 files changed, 150 insertions(+), 160 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e0055151/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityIndexDeleteListener.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityIndexDeleteListener.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityIndexDeleteListener.java
index cf3f4a6..3d4d661 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityIndexDeleteListener.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityIndexDeleteListener.java
@@ -27,6 +27,7 @@ import org.apache.usergrid.persistence.collection.mvcc.entity.impl.MvccEntityDel
import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
import org.apache.usergrid.persistence.core.entity.EntityVersion;
import org.apache.usergrid.persistence.core.rx.ObservableIterator;
+import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
import org.apache.usergrid.persistence.index.EntityIndex;
import org.apache.usergrid.persistence.index.EntityIndexFactory;
import org.apache.usergrid.persistence.index.IndexScope;
@@ -60,8 +61,8 @@ public class CpEntityIndexDeleteListener {
public Observable<EntityVersion> receive(final MvccEntityDeleteEvent event) {
final CollectionScope collectionScope = event.getCollectionScope();
- final IndexScope indexScope = new IndexScopeImpl(collectionScope.getApplication(), collectionScope.getOwner(), collectionScope.getName());
- final EntityIndex entityIndex = entityIndexFactory.createEntityIndex(indexScope);
+ final IndexScope indexScope = new IndexScopeImpl(collectionScope.getOwner(), collectionScope.getName());
+ final EntityIndex entityIndex = entityIndexFactory.createEntityIndex(new ApplicationScopeImpl( collectionScope.getApplication()));
return Observable.create(new ObservableIterator<CandidateResult>("deleteEsIndexVersions") {
@Override
protected Iterator<CandidateResult> getIterator() {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e0055151/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 777b041..d729c97 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -562,8 +562,7 @@ public class CpEntityManager implements EntityManager {
Set<String> collectionNames = collectionsByUuid.get( uuid );
for ( String coll : collectionNames ) {
- IndexScope indexScope = new IndexScopeImpl(
- applicationScope.getApplication(),
+ IndexScope indexScope = new IndexScopeImpl(
new SimpleId( uuid, ownerType ),
CpNamingUtils.getCollectionScopeNameFromCollectionName( coll ) );
@@ -576,15 +575,13 @@ public class CpEntityManager implements EntityManager {
// deindex from default index scope
- IndexScope defaultIndexScope = new IndexScopeImpl(
- applicationScope.getApplication(),
+ IndexScope defaultIndexScope = new IndexScopeImpl(
applicationScope.getApplication(),
CpNamingUtils.getCollectionScopeNameFromEntityType( entityRef.getType() ) );
batch.deindex(defaultIndexScope, entity );
- IndexScope allTypesIndexScope = new IndexScopeImpl(
- applicationScope.getApplication(),
+ IndexScope allTypesIndexScope = new IndexScopeImpl(
applicationScope.getApplication(),
CpNamingUtils.ALL_TYPES);
@@ -971,8 +968,7 @@ public class CpEntityManager implements EntityManager {
applicationScope.getApplication(),
collectionName );
- IndexScope defaultIndexScope = new IndexScopeImpl(
- applicationScope.getApplication(),
+ IndexScope defaultIndexScope = new IndexScopeImpl(
applicationScope.getApplication(),
CpNamingUtils.getCollectionScopeNameFromEntityType( entityRef.getType() ) );
@@ -2755,10 +2751,7 @@ public class CpEntityManager implements EntityManager {
emf.refreshIndex();
// refresh this Entity Manager's application's index
- IndexScope indexScope = new IndexScopeImpl(
- applicationScope.getApplication(), applicationScope.getApplication(), "dummy" );
-
- EntityIndex ei = managerCache.getEntityIndex( indexScope );
+ EntityIndex ei = managerCache.getEntityIndex( applicationScope );
ei.refresh();
}
@@ -2991,14 +2984,12 @@ public class CpEntityManager implements EntityManager {
// Index the new connection in app|source|type context
IndexScope indexScope = new IndexScopeImpl(
- applicationScope.getApplication(),
sourceEntity.getId(),
CpNamingUtils.getConnectionScopeName( targetEntityType, connType ));
batch.index(indexScope, targetEntity);
// Index the new connection in app|scope|all-types context
IndexScope allTypesIndexScope = new IndexScopeImpl(
- applicationScope.getApplication(),
sourceEntity.getId(),
CpNamingUtils.ALL_TYPES);
@@ -3018,7 +3009,6 @@ public class CpEntityManager implements EntityManager {
// index member into entity collection | type scope
IndexScope collectionIndexScope = new IndexScopeImpl(
- applicationScope.getApplication(),
collectionEntity.getId(),
CpNamingUtils.getCollectionScopeNameFromCollectionName( collName ));
@@ -3026,7 +3016,6 @@ public class CpEntityManager implements EntityManager {
// index member into entity | all-types scope
IndexScope entityAllTypesScope = new IndexScopeImpl(
- applicationScope.getApplication(),
collectionEntity.getId(),
CpNamingUtils.ALL_TYPES);
@@ -3035,7 +3024,6 @@ public class CpEntityManager implements EntityManager {
// index member into application | all-types scope
IndexScope appAllTypesScope = new IndexScopeImpl(
applicationScope.getApplication(),
- applicationScope.getApplication(),
CpNamingUtils.ALL_TYPES);
batch.index(appAllTypesScope, memberEntity);
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e0055151/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index f24f26b..845d798 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -558,22 +558,17 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
// refresh special indexes without calling EntityManager refresh because stack overflow
// system app
- IndexScope sscope = new IndexScopeImpl(
- new SimpleId( SYSTEM_APP_ID, "application"),
- new SimpleId( SYSTEM_APP_ID, "application"), "dummy");
- managerCache.getEntityIndex( sscope ).refresh();
-
+
+ managerCache.getEntityIndex( new ApplicationScopeImpl( new SimpleId( SYSTEM_APP_ID, "application" ) ) )
+ .refresh();
+
// default app
- IndexScope mscope = new IndexScopeImpl(
- new SimpleId( getManagementAppId(), "application"),
- new SimpleId( getManagementAppId(), "application"), "dummy");
- managerCache.getEntityIndex( mscope ).refresh();
+ managerCache.getEntityIndex( new ApplicationScopeImpl( new SimpleId( getManagementAppId(), "application" ) ) )
+ .refresh();
// management app
- IndexScope dscope = new IndexScopeImpl(
- new SimpleId( getDefaultAppId(), "application"),
- new SimpleId( getDefaultAppId(), "application"), "dummy");
- managerCache.getEntityIndex( dscope ).refresh();
+ managerCache.getEntityIndex( new ApplicationScopeImpl( new SimpleId( getDefaultAppId(), "application" ) ) )
+ .refresh();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e0055151/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpNamingUtils.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpNamingUtils.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpNamingUtils.java
index 60cec72..45c39ab 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpNamingUtils.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpNamingUtils.java
@@ -41,47 +41,48 @@ public class CpNamingUtils {
*/
static final String EDGE_CONN_SUFFIX = "zzzconnzzz";
+
static String getCollectionScopeNameFromEntityType( String type ) {
- String csn = EDGE_COLL_SUFFIX + Schema.defaultCollectionName( type );
- return csn.toLowerCase();
- }
+ String csn = EDGE_COLL_SUFFIX + Schema.defaultCollectionName( type );
+ return csn.toLowerCase();
+ }
- static String getCollectionScopeNameFromCollectionName( String name ) {
- String csn = EDGE_COLL_SUFFIX + name ;
- return csn.toLowerCase();
- }
+ static String getCollectionScopeNameFromCollectionName( String name ) {
+ String csn = EDGE_COLL_SUFFIX + name;
+ return csn.toLowerCase();
+ }
- static String getConnectionScopeName( String entityType, String connectionType ) {
- String csn = EDGE_CONN_SUFFIX + connectionType + entityType ;
- return csn.toLowerCase();
- }
+ static String getConnectionScopeName( String entityType, String connectionType ) {
+ String csn = EDGE_CONN_SUFFIX + connectionType + entityType;
+ return csn.toLowerCase();
+ }
- static boolean isCollectionEdgeType( String type ) {
- return type.startsWith( EDGE_COLL_SUFFIX );
- }
- static boolean isConnectionEdgeType( String type ) {
- return type.startsWith( EDGE_CONN_SUFFIX );
- }
+ static boolean isCollectionEdgeType( String type ) {
+ return type.startsWith( EDGE_COLL_SUFFIX );
+ }
+ static boolean isConnectionEdgeType( String type ) {
+ return type.startsWith( EDGE_CONN_SUFFIX );
+ }
+
static public String getConnectionType( String edgeType ) {
- String[] parts = edgeType.split("\\|");
+ String[] parts = edgeType.split( "\\|" );
return parts[1];
}
static public String getCollectionName( String edgeType ) {
- String[] parts = edgeType.split("\\|");
+ String[] parts = edgeType.split( "\\|" );
return parts[1];
}
-
- static String getEdgeTypeFromConnectionType( String connectionType) {
+ static String getEdgeTypeFromConnectionType( String connectionType ) {
if ( connectionType != null ) {
String csn = EDGE_CONN_SUFFIX + "|" + connectionType;
@@ -91,16 +92,15 @@ public class CpNamingUtils {
return null;
}
- static String getEdgeTypeFromCollectionName( String collectionName) {
- if ( collectionName != null ) {
+ static String getEdgeTypeFromCollectionName( String collectionName ) {
+
+ if ( collectionName != null ) {
String csn = EDGE_COLL_SUFFIX + "|" + collectionName;
return csn;
}
-
return null;
}
-
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e0055151/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index ca01ddd..5f595f4 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -396,7 +396,6 @@ public class CpRelationManager implements RelationManager {
String collName = CpNamingUtils.getCollectionName( edge.getType() );
indexScope = new IndexScopeImpl(
- applicationScope.getApplication(),
new SimpleId(sourceEntity.getUuid(), sourceEntity.getType()),
CpNamingUtils.getCollectionScopeNameFromCollectionName( collName ));
@@ -404,7 +403,6 @@ public class CpRelationManager implements RelationManager {
String connName = CpNamingUtils.getCollectionName( edge.getType() );
indexScope = new IndexScopeImpl(
- applicationScope.getApplication(),
new SimpleId(sourceEntity.getUuid(), sourceEntity.getType()),
CpNamingUtils.getConnectionScopeName( cpHeadEntity.getId().getType(), connName ));
}
@@ -414,7 +412,6 @@ public class CpRelationManager implements RelationManager {
// reindex the entity in the source entity's all-types index
indexScope = new IndexScopeImpl(
- applicationScope.getApplication(),
new SimpleId(sourceEntity.getUuid(), sourceEntity.getType()),
CpNamingUtils.ALL_TYPES);
@@ -781,7 +778,6 @@ public class CpRelationManager implements RelationManager {
// remove item from collection index
IndexScope indexScope = new IndexScopeImpl(
- applicationScope.getApplication(),
cpHeadEntity.getId(),
CpNamingUtils.getCollectionScopeNameFromCollectionName( collName ));
@@ -789,7 +785,6 @@ public class CpRelationManager implements RelationManager {
// remove collection from item index
IndexScope itemScope = new IndexScopeImpl(
- applicationScope.getApplication(),
memberEntity.getId(),
CpNamingUtils.getCollectionScopeNameFromCollectionName(
Schema.defaultCollectionName( cpHeadEntity.getId().getType() ) ));
@@ -892,17 +887,15 @@ public class CpRelationManager implements RelationManager {
}
IndexScope indexScope = new IndexScopeImpl(
- applicationScope.getApplication(),
cpHeadEntity.getId(),
CpNamingUtils.getCollectionScopeNameFromCollectionName( collName ));
EntityIndex ei = managerCache.getEntityIndex(applicationScope);
- logger.debug("Searching scope {}:{}:{}",
- new String[] {
- indexScope.getApplication().toString(),
+ logger.debug("Searching scope {}:{}",
+
indexScope.getOwner().toString(),
- indexScope.getName() });
+ indexScope.getName() );
query.setEntityType( collection.getType() );
query = adjustQuery( query );
@@ -1016,7 +1009,6 @@ public class CpRelationManager implements RelationManager {
// Index the new connection in app|source|type context
IndexScope indexScope = new IndexScopeImpl(
- applicationScope.getApplication(),
cpHeadEntity.getId(),
CpNamingUtils.getConnectionScopeName( connectedEntityRef.getType(), connectionType ));
@@ -1024,7 +1016,6 @@ public class CpRelationManager implements RelationManager {
// Index the new connection in app|scope|all-types context
IndexScope allTypesIndexScope = new IndexScopeImpl(
- applicationScope.getApplication(),
cpHeadEntity.getId(),
CpNamingUtils.ALL_TYPES);
batch.index(allTypesIndexScope, targetEntity );
@@ -1240,14 +1231,12 @@ public class CpRelationManager implements RelationManager {
// Deindex the connection in app|source|type context
IndexScope indexScope = new IndexScopeImpl(
- applicationScope.getApplication(),
new SimpleId( connectingEntityRef.getUuid(), connectingEntityRef.getType() ),
CpNamingUtils.getConnectionScopeName( targetEntity.getId().getType(), connectionType ));
batch.deindex( indexScope , targetEntity );
// Deindex the connection in app|source|type context
IndexScope allTypesIndexScope = new IndexScopeImpl(
- applicationScope.getApplication(),
new SimpleId( connectingEntityRef.getUuid(), connectingEntityRef.getType() ),
CpNamingUtils.ALL_TYPES);
@@ -1306,17 +1295,15 @@ public class CpRelationManager implements RelationManager {
}
IndexScope indexScope = new IndexScopeImpl(
- applicationScope.getApplication(),
cpHeadEntity.getId(),
scopeName);
final EntityIndex ei = managerCache.getEntityIndex(applicationScope);
- logger.debug("Searching connected entities from scope {}:{}:{}", new String[] {
- indexScope.getApplication().toString(),
+ logger.debug("Searching connected entities from scope {}:{}",
indexScope.getOwner().toString(),
- indexScope.getName()});
+ indexScope.getName());
query = adjustQuery( query );
CandidateResults crs = ei.search( indexScope, query );
@@ -1411,16 +1398,14 @@ public class CpRelationManager implements RelationManager {
// search across all types of collections of the head-entity
IndexScope indexScope = new IndexScopeImpl(
- applicationScope.getApplication(),
cpHeadEntity.getId(),
CpNamingUtils.ALL_TYPES);
EntityIndex ei = managerCache.getEntityIndex(applicationScope);
- logger.debug("Searching connections from the all-types scope {}:{}:{}", new String[] {
- indexScope.getApplication().toString(),
+ logger.debug("Searching connections from the all-types scope {}:{}",
indexScope.getOwner().toString(),
- indexScope.getName()});
+ indexScope.getName());
query = adjustQuery( query );
CandidateResults crs = ei.search(indexScope, query );
@@ -1429,15 +1414,13 @@ public class CpRelationManager implements RelationManager {
}
IndexScope indexScope = new IndexScopeImpl(
- applicationScope.getApplication(),
cpHeadEntity.getId(),
CpNamingUtils.getConnectionScopeName( query.getEntityType(), query.getConnectionType() ));
EntityIndex ei = managerCache.getEntityIndex(applicationScope);
- logger.debug("Searching connections from the '{}' scope {}:{}:{}", new String[] {
- indexScope.getApplication().toString(),
+ logger.debug("Searching connections from the scope {}:{}",
indexScope.getOwner().toString(),
- indexScope.getName()});
+ indexScope.getName());
query = adjustQuery( query );
CandidateResults crs = ei.search( indexScope, query );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e0055151/stack/core/src/test/java/org/apache/usergrid/corepersistence/CpEntityIndexDeleteListenerTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/CpEntityIndexDeleteListenerTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/CpEntityIndexDeleteListenerTest.java
index d59432b..e7675d6 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/CpEntityIndexDeleteListenerTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/CpEntityIndexDeleteListenerTest.java
@@ -28,12 +28,14 @@ import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
+import org.apache.usergrid.Application;
import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.collection.mvcc.entity.MvccEntity;
import org.apache.usergrid.persistence.collection.mvcc.entity.impl.MvccEntityDeleteEvent;
import org.apache.usergrid.persistence.collection.mvcc.entity.impl.MvccEntityImpl;
import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
import org.apache.usergrid.persistence.core.entity.EntityVersion;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.index.EntityIndex;
import org.apache.usergrid.persistence.index.EntityIndexBatch;
import org.apache.usergrid.persistence.index.EntityIndexFactory;
@@ -81,7 +83,7 @@ public class CpEntityIndexDeleteListenerTest {
when(scope.getOwner()).thenReturn(entityId);
when(scope.getName()).thenReturn("test");
when(scope.getApplication()).thenReturn(entityId);
- when(eif.createEntityIndex(any(IndexScope.class))).thenReturn(entityIndex);
+ when(eif.createEntityIndex(any(ApplicationScope.class))).thenReturn(entityIndex);
final EntityIndexBatch batch = mock(EntityIndexBatch.class);
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e0055151/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java b/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
index acfb1e0..36113e1 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
@@ -42,6 +42,8 @@ import java.util.UUID;
import org.apache.usergrid.cassandra.Concurrent;
import org.apache.usergrid.corepersistence.CpEntityManagerFactory;
import org.apache.usergrid.corepersistence.CpSetup;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
import org.apache.usergrid.persistence.index.EntityIndex;
import org.apache.usergrid.persistence.index.EntityIndexFactory;
import org.apache.usergrid.persistence.index.IndexScope;
@@ -224,8 +226,9 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT {
EntityIndexFactory eif = injector.getInstance( EntityIndexFactory.class );
Id appId = new SimpleId( appUuid, "application");
- IndexScope is = new IndexScopeImpl( appId, appId, "application");
- EntityIndex ei = eif.createEntityIndex(is);
+ ApplicationScope scope = new ApplicationScopeImpl( appId );
+ IndexScope is = new IndexScopeImpl( appId, "application");
+ EntityIndex ei = eif.createEntityIndex(scope);
EsEntityIndexImpl eeii = (EsEntityIndexImpl)ei;
eeii.deleteIndex();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e0055151/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/CollectionScopeImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/CollectionScopeImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/CollectionScopeImpl.java
index ce74f6a..9e96230 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/CollectionScopeImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/CollectionScopeImpl.java
@@ -68,12 +68,12 @@ public class CollectionScopeImpl extends ApplicationScopeImpl implements Collect
return false;
}
- final CollectionScopeImpl that = ( CollectionScopeImpl ) o;
+ final CollectionScope that = ( CollectionScope ) o;
- if ( !name.equals( that.name ) ) {
+ if ( !name.equals( that.getName() ) ) {
return false;
}
- if ( !ownerId.equals( that.ownerId ) ) {
+ if ( !ownerId.equals( that.getOwner() ) ) {
return false;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e0055151/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexScope.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexScope.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexScope.java
index e2361e3..9fbb7f5 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexScope.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexScope.java
@@ -23,7 +23,7 @@ import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.model.entity.Id;
-public interface IndexScope extends ApplicationScope {
+public interface IndexScope {
/**
* @return The name of the index. If you use pluralization for you names vs types,
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e0055151/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
index 93f0e41..a1a15dd 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
@@ -27,6 +27,7 @@ import java.util.Map;
import java.util.Set;
import java.util.UUID;
+import org.elasticsearch.action.bulk.BulkItemResponse;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.client.AdminClient;
@@ -114,13 +115,13 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
public EntityIndexBatch index( final IndexScope indexScope, final Entity entity ) {
- IndexValidationUtils.validateScopeMatch( indexScope, applicationScope );
+ IndexValidationUtils.validateIndexScope( indexScope );
final String indexType = createCollectionScopeTypeName( indexScope );
if ( log.isDebugEnabled() ) {
log.debug( "Indexing entity {}:{} in scope\n app {}\n owner {}\n name {}\n type {}", new Object[] {
- entity.getId().getType(), entity.getId().getUuid(), indexScope.getApplication(),
+ entity.getId().getType(), entity.getId().getUuid(), applicationScope.getApplication(),
indexScope.getOwner(), indexScope.getName(), indexType
} );
}
@@ -153,13 +154,13 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
@Override
public EntityIndexBatch deindex( final IndexScope indexScope, final Id id, final UUID version ) {
- IndexValidationUtils.validateScopeMatch( indexScope, applicationScope );
+ IndexValidationUtils.validateIndexScope( indexScope );
final String indexType = createCollectionScopeTypeName( indexScope );
if ( log.isDebugEnabled() ) {
log.debug( "De-indexing entity {}:{} in scope\n app {}\n owner {}\n name {} type {}", new Object[] {
- id.getType(), id.getUuid(), indexScope.getApplication(), indexScope.getOwner(),
+ id.getType(), id.getUuid(), applicationScope.getApplication(), indexScope.getOwner(),
indexScope.getName(), indexType
} );
}
@@ -201,10 +202,19 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
* Execute the request, check for errors, then re-init the batch for future use
*/
private void execute( final BulkRequestBuilder request ) {
- final BulkResponse response = request.execute().actionGet();
- if ( response.hasFailures() ) {
- throw new RuntimeException( "Unable to index documents. Errors are :" + response.buildFailureMessage() );
+ //nothing to do, we haven't added anthing to the index
+ if(request.numberOfActions() == 0){
+ return;
+ }
+
+ final BulkResponse responses = request.execute().actionGet();
+
+ for ( BulkItemResponse response : responses ) {
+ if ( response.isFailed() ) {
+ throw new RuntimeException(
+ "Unable to index documents. Errors are :" + response.getFailure().getMessage() );
+ }
}
initBatch();
@@ -259,7 +269,8 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
}
catch ( IOException ex ) {
throw new RuntimeException(
- "Exception initing type " + typeName + " in app " + indexScope.getApplication().toString() );
+ "Exception initializing type " + typeName + " in app " + applicationScope.getApplication()
+ .toString() );
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e0055151/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexScopeImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexScopeImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexScopeImpl.java
index 738e734..45cc8dd 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexScopeImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexScopeImpl.java
@@ -24,13 +24,11 @@ import org.apache.usergrid.persistence.model.entity.Id;
public class IndexScopeImpl implements IndexScope {
- private final Id appId;
private final Id ownerId;
private final String type;
- public IndexScopeImpl( final Id appId, final Id ownerId, final String type ) {
- this.appId = appId;
+ public IndexScopeImpl( final Id ownerId, final String type ) {
this.ownerId = ownerId;
this.type = type;
@@ -50,8 +48,4 @@ public class IndexScopeImpl implements IndexScope {
}
- @Override
- public Id getApplication() {
- return appId;
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e0055151/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
index d607700..8dbaa0d 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
@@ -54,11 +54,8 @@ public class IndexingUtils {
*/
public static String createCollectionScopeTypeName( IndexScope scope ) {
StringBuilder sb = new StringBuilder();
- String sep = DOC_TYPE_SEPARATOR;
- sb.append( scope.getApplication().getUuid() ).append(sep);
- sb.append( scope.getApplication().getType() ).append(sep);
- sb.append( scope.getOwner().getUuid() ).append(sep);
- sb.append( scope.getOwner().getType() ).append(sep);
+ sb.append( scope.getOwner().getUuid() ).append(DOC_TYPE_SEPARATOR);
+ sb.append( scope.getOwner().getType() ).append(DOC_TYPE_SEPARATOR);
sb.append( scope.getName() );
return sb.toString();
}
@@ -74,9 +71,8 @@ public class IndexingUtils {
public static String createIndexName(
String prefix, ApplicationScope applicationScope) {
StringBuilder sb = new StringBuilder();
- String sep = INDEX_NAME_SEPARATOR;
- sb.append( prefix ).append(sep);
- sb.append( applicationScope.getApplication().getUuid() ).append(sep);
+ sb.append( prefix ).append(INDEX_NAME_SEPARATOR);
+ sb.append( applicationScope.getApplication().getUuid() ).append(INDEX_NAME_SEPARATOR);
sb.append( applicationScope.getApplication().getType() );
return sb.toString();
}
@@ -101,9 +97,8 @@ public class IndexingUtils {
*/
public static String createIndexDocId(Id entityId, UUID version) {
StringBuilder sb = new StringBuilder();
- String sep = DOC_ID_SEPARATOR;
- sb.append( entityId.getUuid() ).append(sep);
- sb.append( entityId.getType() ).append(sep);
+ sb.append( entityId.getUuid() ).append(DOC_ID_SEPARATOR);
+ sb.append( entityId.getType() ).append(DOC_ID_SEPARATOR);
sb.append( version.toString() );
return sb.toString();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e0055151/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/utils/IndexValidationUtils.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/utils/IndexValidationUtils.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/utils/IndexValidationUtils.java
index d6080de..7ed546f 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/utils/IndexValidationUtils.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/utils/IndexValidationUtils.java
@@ -47,17 +47,8 @@ public class IndexValidationUtils {
verifyString( scope.getName(), "name" );
- validateApplicationScope( scope );
}
- /**
- * Validate the scope in the index matches the application scope
- * @param indexScope
- * @param scope
- */
- public static void validateScopeMatch(final IndexScope indexScope,final ApplicationScope scope){
- Preconditions.checkArgument( scope.equals( indexScope ) );
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e0055151/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/CorePerformanceIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/CorePerformanceIT.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/CorePerformanceIT.java
index c4f8f44..f2aab2a 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/CorePerformanceIT.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/CorePerformanceIT.java
@@ -32,6 +32,8 @@ import org.apache.usergrid.persistence.collection.EntityCollectionManager;
import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl;
import org.apache.usergrid.persistence.core.cassandra.CassandraRule;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
import org.apache.usergrid.persistence.index.EntityIndex;
import org.apache.usergrid.persistence.index.EntityIndexBatch;
import org.apache.usergrid.persistence.index.EntityIndexFactory;
@@ -54,6 +56,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
/**
* TODO: make CorePerformanceIT configurable, add CHOP markup.
*/
@@ -93,20 +96,26 @@ public class CorePerformanceIT extends BaseIT {
ecmf = injector.getInstance( EntityCollectionManagerFactory.class );
ecif = injector.getInstance( EntityIndexFactory.class );
+ final ApplicationScope scope = new ApplicationScopeImpl( new SimpleId( "application" ) );
+
log.info("Start Data Load");
- List<IndexScope> scopes = loadData();
+
+ List<IndexScope> scopes = loadData(scope);
+
log.info("Finish Data Load");
log.info("Start Data Read");
- readData( scopes );
+
+
+ readData( scope, scopes );
log.info("Finish Data Read");
- runSelectedQueries( scopes );
+ runSelectedQueries( scope, scopes );
}
- private List<IndexScope> loadData() throws InterruptedException {
+ private List<IndexScope> loadData(final ApplicationScope applicationScope) throws InterruptedException {
long time = new Date().getTime();
@@ -118,10 +127,10 @@ public class CorePerformanceIT extends BaseIT {
String appName = "app-" + j + "-" + time;
Id appId = new SimpleId( appName );
- IndexScope indexScope = new IndexScopeImpl( appId, appId, "reviews" );
+ IndexScope indexScope = new IndexScopeImpl( appId, "reviews" );
scopes.add( indexScope );
- Thread t = new Thread( new DataLoader( indexScope ) );
+ Thread t = new Thread( new DataLoader( applicationScope, indexScope ) );
t.start();
threads.add( t );
}
@@ -135,12 +144,12 @@ public class CorePerformanceIT extends BaseIT {
}
- private void readData( List<IndexScope> scopes ) throws InterruptedException {
+ private void readData(final ApplicationScope applicationScope, List<IndexScope> scopes ) throws InterruptedException {
List<Thread> threads = new ArrayList<Thread>();
for ( IndexScope scope : scopes ) {
- Thread t = new Thread( new DataReader( scope ));
+ Thread t = new Thread( new DataReader( applicationScope, scope ));
t.start();
threads.add(t);
}
@@ -153,17 +162,18 @@ public class CorePerformanceIT extends BaseIT {
static class DataReader implements Runnable {
- IndexScope indexScope;
+ final ApplicationScope scope;
+ final IndexScope indexScope;
- public DataReader( IndexScope indexScope ) {
+ public DataReader( final ApplicationScope scope, IndexScope indexScope ) {
+ this.scope = scope;
this.indexScope = indexScope;
}
public void run() {
- EntityIndex eci = ecif.createEntityIndex( indexScope );
- EntityCollectionManager ecm = ecmf.createCollectionManager( new CollectionScopeImpl(
- indexScope.getApplication(), indexScope.getOwner(), indexScope.getName() ) );
+ EntityIndex eci = ecif.createEntityIndex( scope);
+ EntityCollectionManager ecm = ecmf.createCollectionManager( new CollectionScopeImpl( scope.getApplication(), indexScope.getOwner(), indexScope.getName() ) );
Query query = Query.fromQL( "review_score > 0"); // get all reviews;
query.withLimit( maxEntities < 1000 ? maxEntities : 1000 );
@@ -192,18 +202,20 @@ public class CorePerformanceIT extends BaseIT {
static class DataLoader implements Runnable {
- IndexScope indexScope;
+ final ApplicationScope applicationScope;
+ final IndexScope indexScope;
- public DataLoader( IndexScope indexScope ) {
+ public DataLoader( final ApplicationScope applicationScope, IndexScope indexScope ) {
+ this.applicationScope = applicationScope;
this.indexScope = indexScope;
}
public void run() {
CollectionScope collectionScope = new CollectionScopeImpl(
- indexScope.getApplication(), indexScope.getOwner(), indexScope.getName() );
+ applicationScope.getApplication(), indexScope.getOwner(), indexScope.getName() );
EntityCollectionManager ecm = ecmf.createCollectionManager(collectionScope );
- EntityIndex eci = ecif.createEntityIndex(indexScope );
+ EntityIndex eci = ecif.createEntityIndex(applicationScope );
FileReader fr;
try {
@@ -255,7 +267,7 @@ public class CorePerformanceIT extends BaseIT {
log.info("Indexed {} reviews in {} / {} ",
new Object[] {
count,
- indexScope.getApplication(),
+ applicationScope,
indexScope.getOwner() } );
}
continue;
@@ -293,14 +305,14 @@ public class CorePerformanceIT extends BaseIT {
}
- public void runSelectedQueries( List<IndexScope> indexScopes ) {
+ public void runSelectedQueries(final ApplicationScope scope, List<IndexScope> indexScopes ) {
for ( IndexScope indexScope : indexScopes ) {
- CollectionScope scope = new CollectionScopeImpl(
- indexScope.getApplication(), indexScope.getOwner(), indexScope.getName() );
- EntityIndex eci = ecif.createEntityIndex(indexScope );
+ CollectionScope collectionScope = new CollectionScopeImpl(
+ scope.getApplication(), indexScope.getOwner(), indexScope.getName() );
+ EntityIndex eci = ecif.createEntityIndex(scope );
// TODO: come up with more and more complex queries for CorePerformanceIT
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e0055151/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java
index 5fb02f2..62910bc 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java
@@ -35,6 +35,8 @@ import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl;
import org.apache.usergrid.persistence.collection.util.EntityUtils;
import org.apache.usergrid.persistence.core.cassandra.CassandraRule;
import org.apache.usergrid.persistence.core.cassandra.ITRunner;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
import org.apache.usergrid.persistence.index.EntityIndex;
import org.apache.usergrid.persistence.index.EntityIndexBatch;
import org.apache.usergrid.persistence.index.EntityIndexFactory;
@@ -76,6 +78,7 @@ public class EntityConnectionIndexImplTest extends BaseIT {
public void testBasicOperation() throws IOException {
Id appId = new SimpleId( "application" );
+ ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
// create a muffin
CollectionScope muffinScope = new CollectionScopeImpl( appId, appId, "muffins" );
@@ -103,9 +106,9 @@ public class EntityConnectionIndexImplTest extends BaseIT {
// index connection of "person Dave likes Large Blueberry muffin"
- IndexScope scope = new IndexScopeImpl( appId, person.getId(), "likes" );
+ IndexScope scope = new IndexScopeImpl( person.getId(), "likes" );
- EntityIndex personLikesIndex = ecif.createEntityIndex( scope );
+ EntityIndex personLikesIndex = ecif.createEntityIndex( applicationScope );
EntityIndexBatch batch = personLikesIndex.createBatch();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e0055151/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
index e01c45d..78cb1ae 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
@@ -41,6 +41,8 @@ import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl;
import org.apache.usergrid.persistence.collection.util.EntityUtils;
import org.apache.usergrid.persistence.core.cassandra.CassandraRule;
import org.apache.usergrid.persistence.core.cassandra.ITRunner;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
import org.apache.usergrid.persistence.index.EntityIndex;
import org.apache.usergrid.persistence.index.EntityIndexBatch;
import org.apache.usergrid.persistence.index.EntityIndexFactory;
@@ -93,10 +95,12 @@ public class EntityIndexTest extends BaseIT {
Id appId = new SimpleId( "application" );
- IndexScope indexScope = new IndexScopeImpl( appId, appId, "things" );
+ ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
+ IndexScope indexScope = new IndexScopeImpl( appId, "things" );
- EntityIndex entityIndex = cif.createEntityIndex( indexScope );
+
+ EntityIndex entityIndex = cif.createEntityIndex( applicationScope );
InputStream is = this.getClass().getResourceAsStream( "/sample-large.json" );
ObjectMapper mapper = new ObjectMapper();
@@ -145,9 +149,12 @@ public class EntityIndexTest extends BaseIT {
public void testDeindex() {
Id appId = new SimpleId( "application" );
- IndexScope indexScope = new IndexScopeImpl( appId, appId, "fastcars" );
- EntityIndex entityIndex = cif.createEntityIndex( indexScope );
+ ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
+
+ IndexScope indexScope = new IndexScopeImpl( appId, "fastcars" );
+
+ EntityIndex entityIndex = cif.createEntityIndex( applicationScope );
Map entityMap = new HashMap() {{
put( "name", "Ferrari 212 Inter" );
@@ -269,12 +276,13 @@ public class EntityIndexTest extends BaseIT {
Id appId = new SimpleId( "application" );
Id ownerId = new SimpleId( "owner" );
- IndexScope indexScope = new IndexScopeImpl( appId, ownerId, "user" );
+ ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
- CollectionScope scope = new CollectionScopeImpl( appId, ownerId, "user" );
+ IndexScope indexScope = new IndexScopeImpl( ownerId, "user" );
- EntityIndex entityIndex = cif.createEntityIndex( indexScope );
+
+ EntityIndex entityIndex = cif.createEntityIndex( applicationScope );
final String middleName = "middleName" + UUIDUtils.newTimeUUID();
Map<String, Object> properties = new LinkedHashMap<String, Object>();
@@ -319,9 +327,11 @@ public class EntityIndexTest extends BaseIT {
Id appId = new SimpleId( "application" );
Id ownerId = new SimpleId( "owner" );
- IndexScope appScope = new IndexScopeImpl( appId, ownerId, "user" );
+ ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
+
+ IndexScope appScope = new IndexScopeImpl( ownerId, "user" );
- EntityIndex ei = cif.createEntityIndex( appScope );
+ EntityIndex ei = cif.createEntityIndex( applicationScope );
final String middleName = "middleName" + UUIDUtils.newTimeUUID();
@@ -360,9 +370,11 @@ public class EntityIndexTest extends BaseIT {
Id appId = new SimpleId( "entityindextest" );
Id ownerId = new SimpleId( "multivaluedtype" );
- IndexScope appScope = new IndexScopeImpl( appId, ownerId, "user" );
+ ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
+
+ IndexScope appScope = new IndexScopeImpl( ownerId, "user" );
- EntityIndex ei = cif.createEntityIndex( appScope );
+ EntityIndex ei = cif.createEntityIndex( applicationScope );
// Bill has favorites as string, age as string and retirement goal as number
Map billMap = new HashMap() {{
[09/13] git commit: Updated batching calls
Posted by to...@apache.org.
Updated batching calls
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/c051fdae
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/c051fdae
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/c051fdae
Branch: refs/heads/two-dot-o
Commit: c051fdaecf3c7e86ce6188a7f56dd2310e1d1ec1
Parents: ae9974d
Author: Todd Nine <to...@apache.org>
Authored: Wed Oct 1 23:44:03 2014 -0600
Committer: Todd Nine <to...@apache.org>
Committed: Wed Oct 1 23:44:03 2014 -0600
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 55 ++++++++++++--------
.../corepersistence/CpRelationManager.java | 21 +++++---
.../index/impl/EsEntityIndexImpl.java | 7 +--
3 files changed, 51 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c051fdae/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 5ce871b..8b8dc8d 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -626,7 +626,7 @@ public class CpEntityManager implements EntityManager {
logger.debug( "Deleting indexes of all {} collections owning the entity",
owners.keySet().size() );
- final EntityIndex ei = managerCache.getEntityIndex( appScope );
+ final EntityIndex ei = managerCache.getEntityIndex( applicationScope );
final EntityIndexBatch batch = ei.createBatch();
@@ -1054,7 +1054,7 @@ public class CpEntityManager implements EntityManager {
getCollectionScopeNameFromEntityType( entityRef.getType()) );
EntityCollectionManager ecm = managerCache.getEntityCollectionManager( collectionScope );
- EntityIndex ei = managerCache.getEntityIndex( appScope );
+ EntityIndex ei = managerCache.getEntityIndex( applicationScope );
Id entityId = new SimpleId( entityRef.getUuid(), entityRef.getType() );
@@ -2611,12 +2611,12 @@ public class CpEntityManager implements EntityManager {
}
// Index CP entity into default collection scope
- IndexScope defaultIndexScope = new IndexScopeImpl(
- applicationScope.getApplication(),
- applicationScope.getApplication(),
- CpEntityManager.getCollectionScopeNameFromEntityType( entity.getType() ) );
- EntityIndex ei = managerCache.getEntityIndex( defaultIndexScope );
- ei.index( cpEntity );
+// IndexScope defaultIndexScope = new IndexScopeImpl(
+// applicationScope.getApplication(),
+// applicationScope.getApplication(),
+// CpEntityManager.getCollectionScopeNameFromEntityType( entity.getType() ) );
+// EntityIndex ei = managerCache.getEntityIndex( applicationScope );
+// ei.createBatch().index( defaultIndexScope, cpEntity ).execute();
// reflect changes in the legacy Entity
entity.setUuid( cpEntity.getId().getUuid() );
@@ -3033,7 +3033,7 @@ public class CpEntityManager implements EntityManager {
collName, collectionEntity.getId().getType(), collectionEntity.getId().getUuid(),
memberEntity.getId().getType(), memberEntity.getId().getUuid() });
- indexEntityIntoCollection( collectionEntity, memberEntity, collName );
+ indexEntityIntoCollection( collectionEntity, memberEntity, collName);
CollectionInfo collection = getDefaultSchema()
.getCollection( memberEntity.getId().getType(), collName);
@@ -3053,58 +3053,71 @@ public class CpEntityManager implements EntityManager {
org.apache.usergrid.persistence.model.entity.Entity sourceEntity,
org.apache.usergrid.persistence.model.entity.Entity targetEntity,
String targetEntityType,
- String connType ) {
+ String connType) {
logger.debug("Indexing into connection {} source {}:{} target {}:{}", new Object[] {
connType, sourceEntity.getId().getType(), sourceEntity.getId().getUuid(),
targetEntity.getId().getType(), targetEntity.getId().getUuid() });
+
+ final EntityIndex ei = getManagerCache().getEntityIndex( applicationScope );
+ final EntityIndexBatch batch = ei.createBatch();
+
+
+
+
// Index the new connection in app|source|type context
IndexScope indexScope = new IndexScopeImpl(
applicationScope.getApplication(),
sourceEntity.getId(),
CpEntityManager.getConnectionScopeName(targetEntityType, connType));
- EntityIndex ei = managerCache.getEntityIndex(indexScope);
- ei.index(targetEntity);
+ batch.index(indexScope, targetEntity);
// Index the new connection in app|scope|all-types context
IndexScope allTypesIndexScope = new IndexScopeImpl(
applicationScope.getApplication(),
sourceEntity.getId(),
ALL_TYPES);
- EntityIndex aei = managerCache.getEntityIndex(allTypesIndexScope);
- aei.index(targetEntity);
+
+ batch.index(allTypesIndexScope, targetEntity);
+
+ batch.execute();
}
void indexEntityIntoCollection(
org.apache.usergrid.persistence.model.entity.Entity collectionEntity,
org.apache.usergrid.persistence.model.entity.Entity memberEntity,
- String collName ) {
+ String collName) {
+
+ final EntityIndex ei = getManagerCache().getEntityIndex( applicationScope );
+ final EntityIndexBatch batch = ei.createBatch();
// index member into entity collection | type scope
IndexScope collectionIndexScope = new IndexScopeImpl(
applicationScope.getApplication(),
collectionEntity.getId(),
CpEntityManager.getCollectionScopeNameFromCollectionName(collName));
- EntityIndex collectionIndex = managerCache.getEntityIndex(collectionIndexScope);
- collectionIndex.index(memberEntity);
+
+ batch.index(collectionIndexScope, memberEntity);
// index member into entity | all-types scope
IndexScope entityAllTypesScope = new IndexScopeImpl(
applicationScope.getApplication(),
collectionEntity.getId(),
ALL_TYPES);
- EntityIndex entityAllCollectionIndex = managerCache.getEntityIndex(entityAllTypesScope);
- entityAllCollectionIndex.index(memberEntity);
+
+ batch.index(entityAllTypesScope, memberEntity);
// index member into application | all-types scope
IndexScope appAllTypesScope = new IndexScopeImpl(
applicationScope.getApplication(),
applicationScope.getApplication(),
ALL_TYPES);
- EntityIndex allCollectionIndex = managerCache.getEntityIndex(appAllTypesScope);
- allCollectionIndex.index(memberEntity);
+
+ batch.index(appAllTypesScope, memberEntity);
+
+ batch.execute();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c051fdae/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index 0073eb1..14f9359 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -16,6 +16,8 @@
package org.apache.usergrid.corepersistence;
+import static me.prettyprint.hector.api.factory.HFactory.createMutator;
+import static org.apache.usergrid.corepersistence.CpEntityManager.*;
import com.yammer.metrics.annotation.Metered;
import java.nio.ByteBuffer;
import java.util.AbstractMap;
@@ -30,6 +32,7 @@ import java.util.Map;
import java.util.Set;
import java.util.UUID;
+import org.apache.usergrid.persistence.index.EntityIndexBatch;
import org.apache.usergrid.utils.UUIDUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -140,6 +143,11 @@ import static org.apache.usergrid.utils.UUIDUtils.getTimestampInMicros;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.Assert;
+
+import me.prettyprint.hector.api.Keyspace;
+import me.prettyprint.hector.api.beans.DynamicComposite;
+import me.prettyprint.hector.api.beans.HColumn;
+import me.prettyprint.hector.api.mutation.Mutator;
import rx.Observable;
@@ -917,7 +925,7 @@ public class CpRelationManager implements RelationManager {
while ( !satisfied && queryCount++ < maxQueries ) {
- CandidateResults crs = ei.search( query );
+ CandidateResults crs = ei.search( indexScope, query );
if ( results == null ) {
results = buildResults( query, crs, collName );
@@ -1009,23 +1017,24 @@ public class CpRelationManager implements RelationManager {
GraphManager gm = managerCache.getGraphManager(applicationScope);
gm.writeEdge(edge).toBlockingObservable().last();
+ EntityIndex ei = managerCache.getEntityIndex(applicationScope);
+ EntityIndexBatch batch = ei.createBatch();
+
// Index the new connection in app|source|type context
IndexScope indexScope = new IndexScopeImpl(
applicationScope.getApplication(),
cpHeadEntity.getId(),
CpEntityManager.getConnectionScopeName( connectedEntityRef.getType(), connectionType ));
- EntityIndex ei = managerCache.getEntityIndex(indexScope);
- ei.index( targetEntity );
+
+ batch.index(indexScope, targetEntity );
// Index the new connection in app|scope|all-types context
IndexScope allTypesIndexScope = new IndexScopeImpl(
applicationScope.getApplication(),
cpHeadEntity.getId(),
ALL_TYPES);
- EntityIndex aei = managerCache.getEntityIndex(allTypesIndexScope);
- aei.index( targetEntity );
+ batch.index(allTypesIndexScope, targetEntity );
- batch.index( allTypesIndexScope, targetEntity );
batch.execute();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c051fdae/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 7851afe..264eb1d 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -26,6 +26,7 @@ import java.util.TreeSet;
import java.util.UUID;
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
+import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchScrollRequestBuilder;
@@ -308,9 +309,5 @@ public class EsEntityIndexImpl implements EntityIndex {
}
- @Override
- public void refresh() {
- client.admin().indices().prepareRefresh( indexName ).execute().actionGet();
- log.debug( "Refreshed index: " + indexName );
- }
+
}
[06/13] git commit: Initial hack to add batch operations to ES. Needs
some serious TLC when we have a bit of time.
Posted by to...@apache.org.
Initial hack to add batch operations to ES. Needs some serious TLC when we have a bit of time.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/fba71c2d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/fba71c2d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/fba71c2d
Branch: refs/heads/two-dot-o
Commit: fba71c2d71df58e1bc8dcf2b11f5a3eeb71b052e
Parents: 1e7da5c
Author: Todd Nine <to...@apache.org>
Authored: Wed Oct 1 20:18:26 2014 -0600
Committer: Todd Nine <to...@apache.org>
Committed: Wed Oct 1 20:18:26 2014 -0600
----------------------------------------------------------------------
.../usergrid/persistence/index/EntityIndex.java | 38 +-
.../persistence/index/EntityIndexBatch.java | 71 +++
.../persistence/index/EntityIndexFactory.java | 4 +-
.../index/impl/EsEntityIndexBatchImpl.java | 359 +++++++++++
.../index/impl/EsEntityIndexImpl.java | 633 ++++---------------
.../persistence/index/impl/EsQueryVistor.java | 35 +-
.../persistence/index/impl/IndexingUtils.java | 112 ++++
.../index/impl/CorePerformanceIT.java | 41 +-
.../impl/EntityConnectionIndexImplTest.java | 9 +-
.../persistence/index/impl/EntityIndexTest.java | 110 ++--
10 files changed, 791 insertions(+), 621 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fba71c2d/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
index 2bf1e89..64a3e10 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
@@ -19,56 +19,38 @@
package org.apache.usergrid.persistence.index;
-import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.index.query.Query;
import org.apache.usergrid.persistence.index.query.CandidateResults;
import org.apache.usergrid.persistence.model.entity.Id;
-import java.util.UUID;
-import org.apache.usergrid.persistence.index.query.CandidateResult;
-
/**
* Provides indexing of Entities within a scope.
*/
public interface EntityIndex {
- /**
- * Create index for Entity
- * @param entity Entity to be indexed.
- */
- public void index( Entity entity );
-
/**
- * Remove index of entity.
- * @param entity Entity to be removed from index.
+ * Create the index batch
+ * @return
*/
- public void deindex( Entity entity );
+ public EntityIndexBatch createBatch();
/**
- * Remove index of entity.
- * @param result CandidateResult to be removed from index.
+ * Execute query in Usergrid syntax.
*/
- public void deindex( CandidateResult result );
- /**
- * Remove index of entity.
- * @param id Id to be removed from index.
- * @param version Version to be removed from index.
- */
- public void deindex( Id id, UUID version);
+ public CandidateResults search(final IndexScope indexScope, Query query );
/**
- * Execute query in Usergrid syntax.
+ * Get the candidate results of all versions of the entity for this id
+ * @param id
+ * @return
*/
-
- public CandidateResults search( Query query );
+ public CandidateResults getEntityVersions(final IndexScope indexScope, Id id);
/**
- * Force refresh of index (should be used for testing purposes only).
+ * Refresh the index
*/
public void refresh();
- public CandidateResults getEntityVersions(Id id);
-
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fba71c2d/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexBatch.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexBatch.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexBatch.java
new file mode 100644
index 0000000..643174c
--- /dev/null
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexBatch.java
@@ -0,0 +1,71 @@
+package org.apache.usergrid.persistence.index;/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+import java.util.UUID;
+
+import org.apache.usergrid.persistence.index.query.CandidateResult;
+import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+
+public interface EntityIndexBatch {
+
+
+
+
+ /**
+ * Create index for Entity
+ * @param indexScope The scope for the index
+ * @param entity Entity to be indexed.
+ */
+ public EntityIndexBatch index( final IndexScope indexScope, final Entity entity );
+
+ /**
+ * Remove index of entity
+ * @param scope The scope for the entity
+ * @param entity Entity to be removed from index.
+ */
+ public EntityIndexBatch deindex(final IndexScope scope, final Entity entity );
+
+ /**
+ * Remove index of entity.
+ * @param scope The scope to use for removal
+ * @param result CandidateResult to be removed from index.
+ */
+ public EntityIndexBatch deindex(final IndexScope scope, final CandidateResult result );
+
+ /**
+ * Remove index of entity.
+ * @param scope The scope to remove
+ * @param id Id to be removed from index.
+ * @param version Version to be removed from index.
+ */
+ public EntityIndexBatch deindex(final IndexScope scope, final Id id, final UUID version);
+
+ /**
+ * Execute the batch
+ */
+ public void execute();
+
+ /**
+ * Execute the batch and force the refresh
+ */
+ public void executeAndRefresh();
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fba71c2d/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexFactory.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexFactory.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexFactory.java
index 57ae6a5..1a97b5a 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexFactory.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexFactory.java
@@ -20,11 +20,13 @@ package org.apache.usergrid.persistence.index;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+
import com.google.inject.assistedinject.Assisted;
public interface EntityIndexFactory {
public EntityIndex createEntityIndex(
- @Assisted IndexScope appScope);
+ @Assisted ApplicationScope appScope);
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fba71c2d/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
new file mode 100644
index 0000000..d9857f2
--- /dev/null
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
@@ -0,0 +1,359 @@
+package org.apache.usergrid.persistence.index.impl;/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+
+import org.elasticsearch.action.bulk.BulkRequestBuilder;
+import org.elasticsearch.action.bulk.BulkResponse;
+import org.elasticsearch.client.AdminClient;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.indices.IndexAlreadyExistsException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.core.util.ValidationUtils;
+import org.apache.usergrid.persistence.index.EntityIndexBatch;
+import org.apache.usergrid.persistence.index.IndexFig;
+import org.apache.usergrid.persistence.index.IndexScope;
+import org.apache.usergrid.persistence.index.query.CandidateResult;
+import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.entity.Id;
+import org.apache.usergrid.persistence.model.field.ArrayField;
+import org.apache.usergrid.persistence.model.field.BooleanField;
+import org.apache.usergrid.persistence.model.field.DoubleField;
+import org.apache.usergrid.persistence.model.field.EntityObjectField;
+import org.apache.usergrid.persistence.model.field.Field;
+import org.apache.usergrid.persistence.model.field.FloatField;
+import org.apache.usergrid.persistence.model.field.IntegerField;
+import org.apache.usergrid.persistence.model.field.ListField;
+import org.apache.usergrid.persistence.model.field.LocationField;
+import org.apache.usergrid.persistence.model.field.LongField;
+import org.apache.usergrid.persistence.model.field.SetField;
+import org.apache.usergrid.persistence.model.field.StringField;
+import org.apache.usergrid.persistence.model.field.UUIDField;
+import org.apache.usergrid.persistence.model.field.value.EntityObject;
+
+import com.google.common.base.Joiner;
+
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ANALYZED_STRING_PREFIX;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.BOOLEAN_PREFIX;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ENTITYID_FIELDNAME;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.GEO_PREFIX;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.NUMBER_PREFIX;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.STRING_PREFIX;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.createCollectionScopeTypeName;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.createIndexDocId;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.createIndexName;
+import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+
+
+public class EsEntityIndexBatchImpl implements EntityIndexBatch {
+
+ private static final Logger log = LoggerFactory.getLogger( EsEntityIndexBatchImpl.class );
+
+ private final ApplicationScope applicationScope;
+
+ private final Client client;
+
+ // Keep track of what types we have already initialized to avoid cost
+ // of attempting to init them again. Used in the initType() method.
+ private final Set<String> knownTypes;
+
+ private final boolean refresh;
+
+ private final String indexName;
+
+ private BulkRequestBuilder bulkRequest;
+
+
+ public EsEntityIndexBatchImpl( final ApplicationScope applicationScope,
+ final Client client, final IndexFig config, final Set<String> knownTypes ) {
+
+ this.applicationScope = applicationScope;
+ this.client = client;
+ this.knownTypes = knownTypes;
+ this.indexName = createIndexName( config.getIndexPrefix(), applicationScope );
+ this.refresh = config.isForcedRefresh();
+ initBatch();
+ }
+
+
+ @Override
+ public EntityIndexBatch index( final IndexScope indexScope, final Entity entity ) {
+
+ final String indexType = createCollectionScopeTypeName( indexScope );
+
+ if ( log.isDebugEnabled() ) {
+ log.debug( "Indexing entity {}:{} in scope\n app {}\n owner {}\n name {}\n type {}", new Object[] {
+ entity.getId().getType(), entity.getId().getUuid(), indexScope.getApplication(),
+ indexScope.getOwner(), indexScope.getName(), indexType
+ } );
+ }
+
+ ValidationUtils.verifyEntityWrite( entity );
+
+ initType(indexScope, indexType );
+
+ Map<String, Object> entityAsMap = entityToMap( entity );
+
+ // need prefix here becuase we index UUIDs as strings
+ entityAsMap.put( STRING_PREFIX + ENTITYID_FIELDNAME, entity.getId().getUuid().toString().toLowerCase() );
+
+ // let caller add these fields if needed
+ // entityAsMap.put("created", entity.getId().getUuid().timestamp();
+ // entityAsMap.put("updated", entity.getVersion().timestamp());
+
+ String indexId = createIndexDocId( entity );
+
+ log.debug( "Indexing entity id {} data {} ", indexId, entityAsMap );
+
+ bulkRequest.add( client.prepareIndex( indexName, indexType, indexId ).setSource( entityAsMap ) );
+
+ return this;
+ }
+
+
+ @Override
+ public EntityIndexBatch deindex(final IndexScope indexScope, final Id id, final UUID version ) {
+
+ final String indexType = createCollectionScopeTypeName( indexScope );
+
+ if ( log.isDebugEnabled() ) {
+ log.debug( "De-indexing entity {}:{} in scope\n app {}\n owner {}\n name {} type {}", new Object[] {
+ id.getType(), id.getUuid(), indexScope.getApplication(), indexScope.getOwner(),
+ indexScope.getName(), indexType
+ } );
+ }
+
+ String indexId = createIndexDocId( id, version );
+
+ bulkRequest.add( client.prepareDelete( indexName, indexType, indexId ).setRefresh( refresh ) );
+
+
+ log.debug( "Deindexed Entity with index id " + indexId );
+
+ return this;
+ }
+
+
+ @Override
+ public EntityIndexBatch deindex( final IndexScope indexScope, final Entity entity ) {
+
+ return deindex(indexScope, entity.getId(), entity.getVersion() );
+ }
+
+
+ @Override
+ public EntityIndexBatch deindex( final IndexScope indexScope, final CandidateResult entity ) {
+
+ return deindex(indexScope, entity.getId(), entity.getVersion() );
+ }
+
+
+ @Override
+ public void execute() {
+ execute( bulkRequest.setRefresh( refresh ) );
+ }
+
+
+ /**
+ * Execute the request, check for errors, then re-init the batch for future use
+ * @param request
+ */
+ private void execute( final BulkRequestBuilder request ) {
+ final BulkResponse response = request.execute().actionGet();
+
+ if ( response.hasFailures() ) {
+ throw new RuntimeException( "Unable to index documents. Errors are :" + response.buildFailureMessage() );
+ }
+
+ initBatch();
+ }
+
+
+ @Override
+ public void executeAndRefresh() {
+ execute(bulkRequest.setRefresh( true ) );
+ }
+
+
+ /**
+ * Create ElasticSearch mapping for each type of Entity.
+ */
+ private void initType( final IndexScope indexScope, String typeName ) {
+
+ // no need for synchronization here, it's OK if we init attempt to init type multiple times
+ if ( knownTypes.contains( typeName ) ) {
+ return;
+ }
+
+ AdminClient admin = client.admin();
+ try {
+ XContentBuilder mxcb = EsEntityIndexImpl.createDoubleStringIndexMapping( jsonBuilder(), typeName );
+
+ admin.indices().preparePutMapping( indexName ).setType( typeName ).setSource( mxcb ).execute().actionGet();
+
+ admin.indices().prepareGetMappings( indexName ).addTypes( typeName ).execute().actionGet();
+
+ // log.debug("Created new type mapping");
+ // log.debug(" Scope application: " + indexScope.getApplication());
+ // log.debug(" Scope owner: " + indexScope.getOwner());
+ // log.debug(" Type name: " + typeName);
+
+ knownTypes.add( typeName );
+ }
+ catch ( IndexAlreadyExistsException ignored ) {
+ // expected
+ }
+ catch ( IOException ex ) {
+ throw new RuntimeException(
+ "Exception initing type " + typeName + " in app " + indexScope.getApplication().toString() );
+ }
+ }
+
+
+ /**
+ * Convert Entity to Map. Adding prefixes for types:
+ *
+ * su_ - String unanalyzed field sa_ - String analyzed field go_ - Location field nu_ - Number field bu_ - Boolean
+ * field
+ */
+ private static Map entityToMap( EntityObject entity ) {
+
+ Map<String, Object> entityMap = new HashMap<String, Object>();
+
+ for ( Object f : entity.getFields().toArray() ) {
+
+ Field field = ( Field ) f;
+
+ if ( f instanceof ListField ) {
+ List list = ( List ) field.getValue();
+ entityMap.put( field.getName().toLowerCase(), new ArrayList( processCollectionForMap( list ) ) );
+
+ if ( !list.isEmpty() ) {
+ if ( list.get( 0 ) instanceof String ) {
+ Joiner joiner = Joiner.on( " " ).skipNulls();
+ String joined = joiner.join( list );
+ entityMap.put( ANALYZED_STRING_PREFIX + field.getName().toLowerCase(),
+ new ArrayList( processCollectionForMap( list ) ) );
+ }
+ }
+ }
+ else if ( f instanceof ArrayField ) {
+ List list = ( List ) field.getValue();
+ entityMap.put( field.getName().toLowerCase(), new ArrayList( processCollectionForMap( list ) ) );
+ }
+ else if ( f instanceof SetField ) {
+ Set set = ( Set ) field.getValue();
+ entityMap.put( field.getName().toLowerCase(), new ArrayList( processCollectionForMap( set ) ) );
+ }
+ else if ( f instanceof EntityObjectField ) {
+ EntityObject eo = ( EntityObject ) field.getValue();
+ entityMap.put( field.getName().toLowerCase(), entityToMap( eo ) ); // recursion
+
+ // Add type information as field-name prefixes
+
+ }
+ else if ( f instanceof StringField ) {
+
+ // index in lower case because Usergrid queries are case insensitive
+ entityMap.put( ANALYZED_STRING_PREFIX + field.getName().toLowerCase(),
+ ( ( String ) field.getValue() ).toLowerCase() );
+ entityMap.put( STRING_PREFIX + field.getName().toLowerCase(),
+ ( ( String ) field.getValue() ).toLowerCase() );
+ }
+ else if ( f instanceof LocationField ) {
+ LocationField locField = ( LocationField ) f;
+ Map<String, Object> locMap = new HashMap<String, Object>();
+
+ // field names lat and lon trigger ElasticSearch geo location
+ locMap.put( "lat", locField.getValue().getLatitude() );
+ locMap.put( "lon", locField.getValue().getLongtitude() );
+ entityMap.put( GEO_PREFIX + field.getName().toLowerCase(), locMap );
+ }
+ else if ( f instanceof DoubleField || f instanceof FloatField || f instanceof IntegerField
+ || f instanceof LongField ) {
+
+ entityMap.put( NUMBER_PREFIX + field.getName().toLowerCase(), field.getValue() );
+ }
+ else if ( f instanceof BooleanField ) {
+
+ entityMap.put( BOOLEAN_PREFIX + field.getName().toLowerCase(), field.getValue() );
+ }
+ else if ( f instanceof UUIDField ) {
+
+ entityMap.put( STRING_PREFIX + field.getName().toLowerCase(), field.getValue().toString() );
+ }
+ else {
+ entityMap.put( field.getName().toLowerCase(), field.getValue() );
+ }
+ }
+
+ return entityMap;
+ }
+
+
+ private static Collection processCollectionForMap( Collection c ) {
+ if ( c.isEmpty() ) {
+ return c;
+ }
+ List processed = new ArrayList();
+ Object sample = c.iterator().next();
+
+ if ( sample instanceof Entity ) {
+ for ( Object o : c.toArray() ) {
+ Entity e = ( Entity ) o;
+ processed.add( entityToMap( e ) );
+ }
+ }
+ else if ( sample instanceof List ) {
+ for ( Object o : c.toArray() ) {
+ List list = ( List ) o;
+ processed.add( processCollectionForMap( list ) ); // recursion;
+ }
+ }
+ else if ( sample instanceof Set ) {
+ for ( Object o : c.toArray() ) {
+ Set set = ( Set ) o;
+ processed.add( processCollectionForMap( set ) ); // recursion;
+ }
+ }
+ else {
+ for ( Object o : c.toArray() ) {
+ processed.add( o );
+ }
+ }
+ return processed;
+ }
+
+
+ private void initBatch() {
+ this.bulkRequest = client.prepareBulk();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fba71c2d/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 8401e13..077036a 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -18,55 +18,20 @@
package org.apache.usergrid.persistence.index.impl;
-import com.google.common.base.Joiner;
-import com.google.common.util.concurrent.AtomicDouble;
-import com.google.inject.Inject;
-import com.google.inject.assistedinject.Assisted;
import java.io.IOException;
import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
import java.util.List;
-import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
import java.util.UUID;
-import java.util.concurrent.atomic.AtomicLong;
-import org.apache.commons.lang3.time.StopWatch;
-import org.apache.usergrid.persistence.core.util.ValidationUtils;
-import org.apache.usergrid.persistence.index.EntityIndex;
-import org.apache.usergrid.persistence.index.IndexFig;
-import org.apache.usergrid.persistence.index.IndexScope;
-import org.apache.usergrid.persistence.index.query.CandidateResult;
-import org.apache.usergrid.persistence.index.query.CandidateResults;
-import org.apache.usergrid.persistence.index.query.Query;
-import org.apache.usergrid.persistence.index.utils.IndexValidationUtils;
-import org.apache.usergrid.persistence.model.entity.Entity;
-import org.apache.usergrid.persistence.model.entity.Id;
-import org.apache.usergrid.persistence.model.entity.SimpleId;
-import org.apache.usergrid.persistence.model.field.ArrayField;
-import org.apache.usergrid.persistence.model.field.BooleanField;
-import org.apache.usergrid.persistence.model.field.DoubleField;
-import org.apache.usergrid.persistence.model.field.EntityObjectField;
-import org.apache.usergrid.persistence.model.field.Field;
-import org.apache.usergrid.persistence.model.field.FloatField;
-import org.apache.usergrid.persistence.model.field.IntegerField;
-import org.apache.usergrid.persistence.model.field.ListField;
-import org.apache.usergrid.persistence.model.field.LocationField;
-import org.apache.usergrid.persistence.model.field.LongField;
-import org.apache.usergrid.persistence.model.field.SetField;
-import org.apache.usergrid.persistence.model.field.StringField;
-import org.apache.usergrid.persistence.model.field.UUIDField;
-import org.apache.usergrid.persistence.model.field.value.EntityObject;
+
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
-import org.elasticsearch.action.index.IndexRequestBuilder;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchScrollRequestBuilder;
import org.elasticsearch.client.AdminClient;
import org.elasticsearch.client.Client;
import org.elasticsearch.common.xcontent.XContentBuilder;
-import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import org.elasticsearch.index.query.FilterBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.indices.IndexAlreadyExistsException;
@@ -78,19 +43,42 @@ import org.elasticsearch.search.sort.SortOrder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.core.util.ValidationUtils;
+import org.apache.usergrid.persistence.index.EntityIndex;
+import org.apache.usergrid.persistence.index.EntityIndexBatch;
+import org.apache.usergrid.persistence.index.IndexFig;
+import org.apache.usergrid.persistence.index.IndexScope;
+import org.apache.usergrid.persistence.index.query.CandidateResult;
+import org.apache.usergrid.persistence.index.query.CandidateResults;
+import org.apache.usergrid.persistence.index.query.Query;
+import org.apache.usergrid.persistence.model.entity.Id;
+import org.apache.usergrid.persistence.model.entity.SimpleId;
+
+import com.google.inject.Inject;
+import com.google.inject.assistedinject.Assisted;
+
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ANALYZED_STRING_PREFIX;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.BOOLEAN_PREFIX;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.DOC_ID_SEPARATOR_SPLITTER;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ENTITYID_FIELDNAME;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.GEO_PREFIX;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.NUMBER_PREFIX;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.STRING_PREFIX;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.createCollectionScopeTypeName;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.createIndexName;
+
/**
* Implements index using ElasticSearch Java API.
*/
public class EsEntityIndexImpl implements EntityIndex {
- private static final Logger log = LoggerFactory.getLogger(EsEntityIndexImpl.class);
+ private static final Logger log = LoggerFactory.getLogger( EsEntityIndexImpl.class );
private final String indexName;
- private final String indexType;
-
- private final IndexScope indexScope;
+ private final ApplicationScope applicationScope;
private final Client client;
@@ -98,316 +86,94 @@ public class EsEntityIndexImpl implements EntityIndex {
// of attempting to init them again. Used in the initType() method.
private Set<String> knownTypes = new TreeSet<String>();
- private final boolean refresh;
private final int cursorTimeout;
- private final AtomicLong indexedCount = new AtomicLong(0L);
- private final AtomicDouble averageIndexTime = new AtomicDouble(0);
-
- public static final String STRING_PREFIX = "su_";
- public static final String ANALYZED_STRING_PREFIX = "sa_";
- public static final String GEO_PREFIX = "go_";
- public static final String NUMBER_PREFIX = "nu_";
- public static final String BOOLEAN_PREFIX = "bu_";
- public static final String ENTITYID_FIELDNAME = "zzz_entityid_zzz";
+ private final IndexFig config;
- public static final String DOC_ID_SEPARATOR = "|";
- public static final String DOC_ID_SEPARATOR_SPLITTER = "\\|";
- // These are not allowed in document type names: _ . , | #
- public static final String DOC_TYPE_SEPARATOR = "^";
- public static final String DOC_TYPE_SEPARATOR_SPLITTER = "\\^";
-
- public static final String INDEX_NAME_SEPARATOR = "^";
-
-
@Inject
- public EsEntityIndexImpl(
- @Assisted final IndexScope indexScope,
- IndexFig config,
- EsProvider provider
- ) {
+ public EsEntityIndexImpl( @Assisted final ApplicationScope applicationScope, final IndexFig config, final EsProvider provider ) {
- IndexValidationUtils.validateIndexScope( indexScope );
+ ValidationUtils.validateApplicationScope( applicationScope );
try {
- this.indexScope = indexScope;
+ this.applicationScope = applicationScope;
this.client = provider.getClient();
- this.indexName = createIndexName( config.getIndexPrefix(), indexScope);
- this.indexType = createCollectionScopeTypeName( indexScope );
-
- this.refresh = config.isForcedRefresh();
+ this.indexName = createIndexName( config.getIndexPrefix(), applicationScope );
this.cursorTimeout = config.getQueryCursorTimeout();
- } catch ( Exception e ) {
- log.error("Error setting up index", e);
+ this.config = config;
+ }
+ catch ( Exception e ) {
+ log.error( "Error setting up index", e );
throw e;
}
AdminClient admin = client.admin();
try {
- CreateIndexResponse r = admin.indices().prepareCreate(indexName).execute().actionGet();
- log.debug("Created new Index Name [{}] ACK=[{}]", indexName, r.isAcknowledged());
+ CreateIndexResponse r = admin.indices().prepareCreate( indexName ).execute().actionGet();
+ log.debug( "Created new Index Name [{}] ACK=[{}]", indexName, r.isAcknowledged() );
client.admin().indices().prepareRefresh( indexName ).execute().actionGet();
try {
// TODO: figure out what refresh above is not enough to ensure index is ready
- Thread.sleep(500);
- } catch (InterruptedException ex) {}
-
- } catch (IndexAlreadyExistsException ignored) {
- // expected
- }
- }
-
-
- /**
- * Create ElasticSearch mapping for each type of Entity.
- */
- private void initType( String typeName ) {
-
- // no need for synchronization here, it's OK if we init attempt to init type multiple times
- if ( knownTypes.contains( typeName )) {
- return;
- }
-
- AdminClient admin = client.admin();
- try {
- XContentBuilder mxcb = EsEntityIndexImpl
- .createDoubleStringIndexMapping(jsonBuilder(), typeName);
-
- admin.indices().preparePutMapping(indexName)
- .setType(typeName).setSource(mxcb).execute().actionGet();
-
- admin.indices().prepareGetMappings(indexName)
- .addTypes(typeName).execute().actionGet();
-
-// log.debug("Created new type mapping");
-// log.debug(" Scope application: " + indexScope.getApplication());
-// log.debug(" Scope owner: " + indexScope.getOwner());
-// log.debug(" Type name: " + typeName);
-
- knownTypes.add( typeName );
-
- } catch (IndexAlreadyExistsException ignored) {
- // expected
- }
- catch (IOException ex) {
- throw new RuntimeException("Exception initing type " + typeName
- + " in app " + indexScope.getApplication().toString());
- }
- }
-
-
- /**
- * Create the index name based on our prefix+appUUID+AppType
- * @param prefix
- * @param indexScope
- * @return
- */
- private String createIndexName(
- String prefix, IndexScope indexScope) {
- StringBuilder sb = new StringBuilder();
- String sep = INDEX_NAME_SEPARATOR;
- sb.append( prefix ).append(sep);
- sb.append( indexScope.getApplication().getUuid() ).append(sep);
- sb.append( indexScope.getApplication().getType() );
- return sb.toString();
- }
-
-
- /**
- * Create the index doc from the given entity
- * @param entity
- * @return
- */
- private String createIndexDocId(Entity entity) {
- return createIndexDocId(entity.getId(), entity.getVersion());
- }
-
-
- /**
- * Create the doc Id. This is the entitie's type + uuid + version
- * @param entityId
- * @param version
- * @return
- */
- private String createIndexDocId(Id entityId, UUID version) {
- StringBuilder sb = new StringBuilder();
- String sep = DOC_ID_SEPARATOR;
- sb.append( entityId.getUuid() ).append(sep);
- sb.append( entityId.getType() ).append(sep);
- sb.append( version.toString() );
- return sb.toString();
- }
-
-
- /**
- * Create our sub scope. This is the ownerUUID + type
- * @param scope
- * @return
- */
- private static String createCollectionScopeTypeName( IndexScope scope ) {
- StringBuilder sb = new StringBuilder();
- String sep = DOC_TYPE_SEPARATOR;
- sb.append( scope.getApplication().getUuid() ).append(sep);
- sb.append( scope.getApplication().getType() ).append(sep);
- sb.append( scope.getOwner().getUuid() ).append(sep);
- sb.append( scope.getOwner().getType() ).append(sep);
- sb.append( scope.getName() );
- return sb.toString();
- }
-
-
-
- @Override
- public void index( Entity entity ) {
-
- if ( log.isDebugEnabled() ) {
- log.debug("Indexing entity {}:{} in scope\n app {}\n owner {}\n name {}\n type {}",
- new Object[] {
- entity.getId().getType(),
- entity.getId().getUuid(),
- indexScope.getApplication(),
- indexScope.getOwner(),
- indexScope.getName(),
- indexType
- });
- }
-
- ValidationUtils.verifyEntityWrite(entity);
-
- initType( indexType );
-
- StopWatch timer = null;
- if ( log.isDebugEnabled() ) {
- timer = new StopWatch();
- timer.start();
- }
-
- Map<String, Object> entityAsMap = EsEntityIndexImpl.entityToMap(entity);
-
- // need prefix here becuase we index UUIDs as strings
- entityAsMap.put( STRING_PREFIX + ENTITYID_FIELDNAME,
- entity.getId().getUuid().toString().toLowerCase());
-
- // let caller add these fields if needed
- // entityAsMap.put("created", entity.getId().getUuid().timestamp();
- // entityAsMap.put("updated", entity.getVersion().timestamp());
-
- String indexId = EsEntityIndexImpl.this.createIndexDocId(entity);
-
- log.debug("Indexing entity id {} data {} ", indexId, entityAsMap);
-
- IndexRequestBuilder irb = client
- .prepareIndex( indexName, this.indexType, indexId)
- .setSource(entityAsMap)
- .setRefresh(refresh);
-
- irb.execute().actionGet();
-
- if ( refresh) {
- refresh();
- }
-
- //log.debug("Indexed Entity with index id " + indexId);
-
- if ( log.isDebugEnabled() ) {
- timer.stop();
- double average = averageIndexTime.get();
- if ( !averageIndexTime.compareAndSet( 0, timer.getTime() ) ) {
- averageIndexTime.compareAndSet( average, (average + timer.getTime()) / 2.0 );
+ Thread.sleep( 500 );
}
- long count = indexedCount.addAndGet(1);
- if ( count % 1000 == 0 ) {
- log.debug("Indexed {} entities, average time {}ms",
- count, averageIndexTime.get() );
+ catch ( InterruptedException ex ) {
}
}
- }
-
- @Override
- public void deindex( final Id id, final UUID version) {
-
- if ( log.isDebugEnabled() ) {
- log.debug("De-indexing entity {}:{} in scope\n app {}\n owner {}\n name {} type {}",
- new Object[] {
- id.getType(),
- id.getUuid(),
- indexScope.getApplication(),
- indexScope.getOwner(),
- indexScope.getName(),
- indexType
- });
- }
-
- String indexId = createIndexDocId( id, version );
- client.prepareDelete( indexName, indexType, indexId )
- .setRefresh( refresh )
- .execute().actionGet();
-
- if ( refresh ) {
- refresh();
+ catch ( IndexAlreadyExistsException ignored ) {
+ // expected
}
-
- log.debug("Deindexed Entity with index id " + indexId);
}
- @Override
- public void deindex( Entity entity ) {
- deindex( entity.getId(), entity.getVersion() );
- }
@Override
- public void deindex( CandidateResult entity ) {
-
- deindex( entity.getId(), entity.getVersion() );
-
+ public EntityIndexBatch createBatch() {
+ return new EsEntityIndexBatchImpl( applicationScope, client, config, knownTypes );
}
@Override
- public CandidateResults search(Query query) {
+ public CandidateResults search( final IndexScope indexScope, final Query query ) {
+
+ final String indexType = createCollectionScopeTypeName( indexScope );
QueryBuilder qb = query.createQueryBuilder();
if ( log.isDebugEnabled() ) {
- log.debug("Searching index {}\n type {}\n query {} limit {}",
- new Object[] {
- this.indexName,
- this.indexType,
- qb.toString().replace("\n", " "),
- query.getLimit()
- });
+ log.debug( "Searching index {}\n type {}\n query {} limit {}", new Object[] {
+ this.indexName, indexType, qb.toString().replace( "\n", " " ), query.getLimit()
+ } );
}
-
+
SearchResponse searchResponse;
- if (query.getCursor() == null) {
+ if ( query.getCursor() == null ) {
- SearchRequestBuilder srb = client.prepareSearch(indexName)
- .setTypes( indexType )
- .setScroll( cursorTimeout + "m" )
- .setQuery( qb );
+ SearchRequestBuilder srb =
+ client.prepareSearch( indexName ).setTypes( indexType ).setScroll( cursorTimeout + "m" )
+ .setQuery( qb );
FilterBuilder fb = query.createFilterBuilder();
- if (fb != null) {
- log.debug(" Filter: {} ", fb.toString());
- srb = srb.setPostFilter(fb);
+ if ( fb != null ) {
+ log.debug( " Filter: {} ", fb.toString() );
+ srb = srb.setPostFilter( fb );
}
- srb = srb.setFrom(0).setSize(query.getLimit());
+ srb = srb.setFrom( 0 ).setSize( query.getLimit() );
- for (Query.SortPredicate sp : query.getSortPredicates()) {
+ for ( Query.SortPredicate sp : query.getSortPredicates() ) {
final SortOrder order;
- if (sp.getDirection().equals(Query.SortDirection.ASCENDING)) {
+ if ( sp.getDirection().equals( Query.SortDirection.ASCENDING ) ) {
order = SortOrder.ASC;
- } else {
+ }
+ else {
order = SortOrder.DESC;
}
@@ -416,272 +182,117 @@ public class EsEntityIndexImpl implements EntityIndex {
// that you can order by: string, number and boolean and we ask ElasticSearch
// to ignore any fields that are not present.
- final String stringFieldName = STRING_PREFIX + sp.getPropertyName();
- final FieldSortBuilder stringSort = SortBuilders
- .fieldSort( stringFieldName )
- .order(order)
- .ignoreUnmapped(true);
+ final String stringFieldName = STRING_PREFIX + sp.getPropertyName();
+ final FieldSortBuilder stringSort =
+ SortBuilders.fieldSort( stringFieldName ).order( order ).ignoreUnmapped( true );
srb.addSort( stringSort );
- log.debug(" Sort: {} order by {}", stringFieldName, order.toString());
+ log.debug( " Sort: {} order by {}", stringFieldName, order.toString() );
- final String numberFieldName = NUMBER_PREFIX + sp.getPropertyName();
- final FieldSortBuilder numberSort = SortBuilders
- .fieldSort( numberFieldName )
- .order(order)
- .ignoreUnmapped(true);
+ final String numberFieldName = NUMBER_PREFIX + sp.getPropertyName();
+ final FieldSortBuilder numberSort =
+ SortBuilders.fieldSort( numberFieldName ).order( order ).ignoreUnmapped( true );
srb.addSort( numberSort );
- log.debug(" Sort: {} order by {}", numberFieldName, order.toString());
+ log.debug( " Sort: {} order by {}", numberFieldName, order.toString() );
- final String booleanFieldName = BOOLEAN_PREFIX + sp.getPropertyName();
- final FieldSortBuilder booleanSort = SortBuilders
- .fieldSort( booleanFieldName )
- .order(order)
- .ignoreUnmapped(true);
+ final String booleanFieldName = BOOLEAN_PREFIX + sp.getPropertyName();
+ final FieldSortBuilder booleanSort =
+ SortBuilders.fieldSort( booleanFieldName ).order( order ).ignoreUnmapped( true );
srb.addSort( booleanSort );
- log.debug(" Sort: {} order by {}", booleanFieldName, order.toString());
+ log.debug( " Sort: {} order by {}", booleanFieldName, order.toString() );
}
searchResponse = srb.execute().actionGet();
-
- } else {
+ }
+ else {
String scrollId = query.getCursor();
- if ( scrollId.startsWith("\"")) {
- scrollId = scrollId.substring(1);
+ if ( scrollId.startsWith( "\"" ) ) {
+ scrollId = scrollId.substring( 1 );
}
- if ( scrollId.endsWith("\"")) {
- scrollId = scrollId.substring(0, scrollId.length() - 1 );
+ if ( scrollId.endsWith( "\"" ) ) {
+ scrollId = scrollId.substring( 0, scrollId.length() - 1 );
}
- log.debug("Executing query with cursor: {} ", scrollId);
+ log.debug( "Executing query with cursor: {} ", scrollId );
- SearchScrollRequestBuilder ssrb = client
- .prepareSearchScroll(scrollId)
- .setScroll( cursorTimeout + "m" );
+ SearchScrollRequestBuilder ssrb = client.prepareSearchScroll( scrollId ).setScroll( cursorTimeout + "m" );
searchResponse = ssrb.execute().actionGet();
}
SearchHits hits = searchResponse.getHits();
- log.debug(" Hit count: {} Total hits: {}", hits.getHits().length, hits.getTotalHits() );
+ log.debug( " Hit count: {} Total hits: {}", hits.getHits().length, hits.getTotalHits() );
List<CandidateResult> candidates = new ArrayList<CandidateResult>();
- for (SearchHit hit : hits.getHits()) {
+ for ( SearchHit hit : hits.getHits() ) {
String[] idparts = hit.getId().split( DOC_ID_SEPARATOR_SPLITTER );
- String id = idparts[0];
- String type = idparts[1];
+ String id = idparts[0];
+ String type = idparts[1];
String version = idparts[2];
- Id entityId = new SimpleId(UUID.fromString(id), type);
+ Id entityId = new SimpleId( UUID.fromString( id ), type );
- candidates.add( new CandidateResult( entityId, UUID.fromString(version) ));
+ candidates.add( new CandidateResult( entityId, UUID.fromString( version ) ) );
}
CandidateResults candidateResults = new CandidateResults( query, candidates );
if ( candidates.size() >= query.getLimit() ) {
- candidateResults.setCursor(searchResponse.getScrollId());
- log.debug(" Cursor = " + searchResponse.getScrollId() );
- }
-
- return candidateResults;
- }
-
-
- /**
- * Convert Entity to Map. Adding prefixes for types:
- *
- * su_ - String unanalyzed field
- * sa_ - String analyzed field
- * go_ - Location field
- * nu_ - Number field
- * bu_ - Boolean field
- */
- private static Map entityToMap(EntityObject entity) {
-
- Map<String, Object> entityMap = new HashMap<String, Object>();
-
- for (Object f : entity.getFields().toArray()) {
-
- Field field = (Field) f;
-
- if (f instanceof ListField) {
- List list = (List) field.getValue();
- entityMap.put(field.getName().toLowerCase(),
- new ArrayList(processCollectionForMap(list)));
-
- if ( !list.isEmpty() ) {
- if ( list.get(0) instanceof String ) {
- Joiner joiner = Joiner.on(" ").skipNulls();
- String joined = joiner.join(list);
- entityMap.put( ANALYZED_STRING_PREFIX + field.getName().toLowerCase(),
- new ArrayList(processCollectionForMap(list)));
-
- }
- }
-
- } else if (f instanceof ArrayField) {
- List list = (List) field.getValue();
- entityMap.put(field.getName().toLowerCase(),
- new ArrayList(processCollectionForMap(list)));
-
- } else if (f instanceof SetField) {
- Set set = (Set) field.getValue();
- entityMap.put(field.getName().toLowerCase(),
- new ArrayList(processCollectionForMap(set)));
-
- } else if (f instanceof EntityObjectField) {
- EntityObject eo = (EntityObject)field.getValue();
- entityMap.put(field.getName().toLowerCase(), entityToMap(eo)); // recursion
-
- // Add type information as field-name prefixes
-
- } else if (f instanceof StringField) {
-
- // index in lower case because Usergrid queries are case insensitive
- entityMap.put( ANALYZED_STRING_PREFIX + field.getName().toLowerCase(),
- ((String) field.getValue()).toLowerCase());
- entityMap.put( STRING_PREFIX + field.getName().toLowerCase(),
- ((String) field.getValue()).toLowerCase());
-
- } else if (f instanceof LocationField) {
- LocationField locField = (LocationField) f;
- Map<String, Object> locMap = new HashMap<String, Object>();
-
- // field names lat and lon trigger ElasticSearch geo location
- locMap.put("lat", locField.getValue().getLatitude());
- locMap.put("lon", locField.getValue().getLongtitude());
- entityMap.put( GEO_PREFIX + field.getName().toLowerCase(), locMap);
-
- } else if ( f instanceof DoubleField
- || f instanceof FloatField
- || f instanceof IntegerField
- || f instanceof LongField ) {
-
- entityMap.put( NUMBER_PREFIX + field.getName().toLowerCase(), field.getValue());
-
- } else if ( f instanceof BooleanField ) {
-
- entityMap.put( BOOLEAN_PREFIX + field.getName().toLowerCase(), field.getValue());
-
- } else if ( f instanceof UUIDField ) {
-
- entityMap.put( STRING_PREFIX + field.getName().toLowerCase(),
- field.getValue().toString() );
-
- } else {
- entityMap.put(field.getName().toLowerCase(), field.getValue());
- }
- }
-
- return entityMap;
- }
-
-
- private static Collection processCollectionForMap(Collection c) {
- if (c.isEmpty()) {
- return c;
+ candidateResults.setCursor( searchResponse.getScrollId() );
+ log.debug( " Cursor = " + searchResponse.getScrollId() );
}
- List processed = new ArrayList();
- Object sample = c.iterator().next();
-
- if (sample instanceof Entity) {
- for (Object o : c.toArray()) {
- Entity e = (Entity) o;
- processed.add(entityToMap(e));
- }
- } else if (sample instanceof List) {
- for (Object o : c.toArray()) {
- List list = (List) o;
- processed.add(processCollectionForMap(list)); // recursion;
- }
-
- } else if (sample instanceof Set) {
- for (Object o : c.toArray()) {
- Set set = (Set) o;
- processed.add(processCollectionForMap(set)); // recursion;
- }
-
- } else {
- for (Object o : c.toArray()) {
- processed.add(o);
- }
- }
- return processed;
+ return candidateResults;
}
/**
- * Build mappings for data to be indexed. Setup String fields as not_analyzed and analyzed,
- * where the analyzed field is named {name}_ug_analyzed
+ * Build mappings for data to be indexed. Setup String fields as not_analyzed and analyzed, where the analyzed field
+ * is named {name}_ug_analyzed
*
* @param builder Add JSON object to this builder.
* @param type ElasticSearch type of entity.
+ *
* @return Content builder with JSON for mapping.
*
* @throws java.io.IOException On JSON generation error.
*/
- public static XContentBuilder createDoubleStringIndexMapping(
- XContentBuilder builder, String type) throws IOException {
-
- builder = builder
- .startObject()
- .startObject(type)
- .startArray("dynamic_templates")
-
- // any string with field name that starts with sa_ gets analyzed
- .startObject()
- .startObject("template_1")
- .field("match", ANALYZED_STRING_PREFIX + "*")
- .field("match_mapping_type", "string")
- .startObject("mapping")
- .field("type", "string")
- .field("index", "analyzed")
- .endObject()
- .endObject()
- .endObject()
+ public static XContentBuilder createDoubleStringIndexMapping( XContentBuilder builder, String type )
+ throws IOException {
+
+ builder = builder.startObject().startObject( type ).startArray( "dynamic_templates" )
+
+ // any string with field name that starts with sa_ gets analyzed
+ .startObject().startObject( "template_1" ).field( "match", ANALYZED_STRING_PREFIX + "*" )
+ .field( "match_mapping_type", "string" ).startObject( "mapping" ).field( "type", "string" )
+ .field( "index", "analyzed" ).endObject().endObject().endObject()
// all other strings are not analyzed
- .startObject()
- .startObject("template_2")
- .field("match", "*")
- .field("match_mapping_type", "string")
- .startObject("mapping")
- .field("type", "string")
- .field("index", "not_analyzed")
- .endObject()
- .endObject()
- .endObject()
-
+ .startObject().startObject( "template_2" ).field( "match", "*" ).field( "match_mapping_type", "string" )
+ .startObject( "mapping" ).field( "type", "string" ).field( "index", "not_analyzed" ).endObject()
+ .endObject().endObject()
+
// fields names starting with go_ get geo-indexed
- .startObject()
- .startObject("template_3")
- .field("match", GEO_PREFIX + "location")
- .startObject("mapping")
- .field("type", "geo_point")
- .endObject()
- .endObject()
- .endObject()
-
- .endArray()
- .endObject()
- .endObject();
+ .startObject().startObject( "template_3" ).field( "match", GEO_PREFIX + "location" )
+ .startObject( "mapping" ).field( "type", "geo_point" ).endObject().endObject().endObject()
+
+ .endArray().endObject().endObject();
return builder;
}
- public void refresh() {
- client.admin().indices().prepareRefresh( indexName ).execute().actionGet();
- log.debug("Refreshed index: " + indexName);
- }
-
@Override
- public CandidateResults getEntityVersions(Id id) {
+ public CandidateResults getEntityVersions( final IndexScope scope, final Id id ) {
Query query = new Query();
- query.addEqualityFilter(ENTITYID_FIELDNAME,id.getUuid().toString());
- CandidateResults results = search( query );
+ query.addEqualityFilter( ENTITYID_FIELDNAME, id.getUuid().toString() );
+ CandidateResults results = search( scope, query );
return results;
}
+
+ @Override
+ public void refresh() {
+ client.admin().indices().prepareRefresh( indexName ).execute().actionGet();
+ log.debug( "Refreshed index: " + indexName );
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fba71c2d/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java
index 5634183..0afe992 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java
@@ -18,19 +18,24 @@
package org.apache.usergrid.persistence.index.impl;
-import com.google.common.base.Joiner;
+
import java.util.ArrayList;
import java.util.List;
import java.util.Stack;
import java.util.UUID;
+
+import org.elasticsearch.common.unit.DistanceUnit;
+import org.elasticsearch.index.query.BoolQueryBuilder;
+import org.elasticsearch.index.query.FilterBuilder;
+import org.elasticsearch.index.query.FilterBuilders;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.usergrid.persistence.index.exceptions.IndexException;
import org.apache.usergrid.persistence.index.exceptions.NoFullTextIndexException;
import org.apache.usergrid.persistence.index.exceptions.NoIndexException;
-import org.apache.usergrid.persistence.index.exceptions.IndexException;
-import static org.apache.usergrid.persistence.index.impl.EsEntityIndexImpl.ANALYZED_STRING_PREFIX;
-import static org.apache.usergrid.persistence.index.impl.EsEntityIndexImpl.BOOLEAN_PREFIX;
-import static org.apache.usergrid.persistence.index.impl.EsEntityIndexImpl.GEO_PREFIX;
-import static org.apache.usergrid.persistence.index.impl.EsEntityIndexImpl.NUMBER_PREFIX;
-import static org.apache.usergrid.persistence.index.impl.EsEntityIndexImpl.STRING_PREFIX;
import org.apache.usergrid.persistence.index.query.tree.AndOperand;
import org.apache.usergrid.persistence.index.query.tree.ContainsOperand;
import org.apache.usergrid.persistence.index.query.tree.Equal;
@@ -42,14 +47,14 @@ import org.apache.usergrid.persistence.index.query.tree.NotOperand;
import org.apache.usergrid.persistence.index.query.tree.OrOperand;
import org.apache.usergrid.persistence.index.query.tree.QueryVisitor;
import org.apache.usergrid.persistence.index.query.tree.WithinOperand;
-import org.elasticsearch.common.unit.DistanceUnit;
-import org.elasticsearch.index.query.BoolQueryBuilder;
-import org.elasticsearch.index.query.FilterBuilder;
-import org.elasticsearch.index.query.FilterBuilders;
-import org.elasticsearch.index.query.QueryBuilder;
-import org.elasticsearch.index.query.QueryBuilders;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Joiner;
+
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ANALYZED_STRING_PREFIX;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.BOOLEAN_PREFIX;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.GEO_PREFIX;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.NUMBER_PREFIX;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.STRING_PREFIX;
/**
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fba71c2d/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
new file mode 100644
index 0000000..d607700
--- /dev/null
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
@@ -0,0 +1,112 @@
+package org.apache.usergrid.persistence.index.impl;/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+import java.util.UUID;
+
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.index.IndexScope;
+import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+
+public class IndexingUtils {
+
+
+ public static final String STRING_PREFIX = "su_";
+ public static final String ANALYZED_STRING_PREFIX = "sa_";
+ public static final String GEO_PREFIX = "go_";
+ public static final String NUMBER_PREFIX = "nu_";
+ public static final String BOOLEAN_PREFIX = "bu_";
+
+ public static final String ENTITYID_FIELDNAME = "zzz_entityid_zzz";
+
+ public static final String DOC_ID_SEPARATOR = "|";
+ public static final String DOC_ID_SEPARATOR_SPLITTER = "\\|";
+
+ // These are not allowed in document type names: _ . , | #
+ public static final String DOC_TYPE_SEPARATOR = "^";
+ public static final String DOC_TYPE_SEPARATOR_SPLITTER = "\\^";
+
+ public static final String INDEX_NAME_SEPARATOR = "^";
+
+
+ /**
+ * Create our sub scope. This is the ownerUUID + type
+ * @param scope
+ * @return
+ */
+ public static String createCollectionScopeTypeName( IndexScope scope ) {
+ StringBuilder sb = new StringBuilder();
+ String sep = DOC_TYPE_SEPARATOR;
+ sb.append( scope.getApplication().getUuid() ).append(sep);
+ sb.append( scope.getApplication().getType() ).append(sep);
+ sb.append( scope.getOwner().getUuid() ).append(sep);
+ sb.append( scope.getOwner().getType() ).append(sep);
+ sb.append( scope.getName() );
+ return sb.toString();
+ }
+
+
+
+ /**
+ * Create the index name based on our prefix+appUUID+AppType
+ * @param prefix
+ * @param applicationScope
+ * @return
+ */
+ public static String createIndexName(
+ String prefix, ApplicationScope applicationScope) {
+ StringBuilder sb = new StringBuilder();
+ String sep = INDEX_NAME_SEPARATOR;
+ sb.append( prefix ).append(sep);
+ sb.append( applicationScope.getApplication().getUuid() ).append(sep);
+ sb.append( applicationScope.getApplication().getType() );
+ return sb.toString();
+ }
+
+
+
+ /**
+ * Create the index doc from the given entity
+ * @param entity
+ * @return
+ */
+ public static String createIndexDocId(Entity entity) {
+ return createIndexDocId(entity.getId(), entity.getVersion());
+ }
+
+
+ /**
+ * Create the doc Id. This is the entitie's type + uuid + version
+ * @param entityId
+ * @param version
+ * @return
+ */
+ public static String createIndexDocId(Id entityId, UUID version) {
+ StringBuilder sb = new StringBuilder();
+ String sep = DOC_ID_SEPARATOR;
+ sb.append( entityId.getUuid() ).append(sep);
+ sb.append( entityId.getType() ).append(sep);
+ sb.append( version.toString() );
+ return sb.toString();
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fba71c2d/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/CorePerformanceIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/CorePerformanceIT.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/CorePerformanceIT.java
index 9419809..c4f8f44 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/CorePerformanceIT.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/CorePerformanceIT.java
@@ -33,6 +33,7 @@ import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory
import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl;
import org.apache.usergrid.persistence.core.cassandra.CassandraRule;
import org.apache.usergrid.persistence.index.EntityIndex;
+import org.apache.usergrid.persistence.index.EntityIndexBatch;
import org.apache.usergrid.persistence.index.EntityIndexFactory;
import org.apache.usergrid.persistence.index.IndexScope;
import org.apache.usergrid.persistence.index.guice.TestIndexModule;
@@ -167,12 +168,12 @@ public class CorePerformanceIT extends BaseIT {
Query query = Query.fromQL( "review_score > 0"); // get all reviews;
query.withLimit( maxEntities < 1000 ? maxEntities : 1000 );
- CandidateResults candidateResults = eci.search( query );
+ CandidateResults candidateResults = eci.search(indexScope, query );
int count = candidateResults.size();
while ( candidateResults.hasCursor() && count < maxEntities ) {
query.setCursor( candidateResults.getCursor() ) ;
- candidateResults = eci.search( query );
+ candidateResults = eci.search(indexScope, query );
count += candidateResults.size();
//cause retrieval from cassandra
@@ -221,6 +222,9 @@ public class CorePerformanceIT extends BaseIT {
// Id appId = orgAppScope.scope.getOwner();
int count = 0;
+
+ EntityIndexBatch entityIndexBatch = eci.createBatch();
+
try {
while ( (s = br.readLine()) != null && count < maxEntities ) {
@@ -230,7 +234,8 @@ public class CorePerformanceIT extends BaseIT {
// write and index current entity
ecm.write( current ).toBlocking().last();
- eci.index( current );
+
+ entityIndexBatch.index(indexScope, current );
if ( maxEntities < 20 ) {
log.info("Index written for {}", current.getId());
@@ -242,6 +247,10 @@ public class CorePerformanceIT extends BaseIT {
new SimpleId(UUIDGenerator.newTimeUUID(), "review"));
count++;
+ if(count % 1000 == 0){
+ entityIndexBatch.execute();
+ }
+
if (count % 100000 == 0) {
log.info("Indexed {} reviews in {} / {} ",
new Object[] {
@@ -295,23 +304,23 @@ public class CorePerformanceIT extends BaseIT {
// TODO: come up with more and more complex queries for CorePerformanceIT
- query(eci, "product_productid = 'B006K2ZZ7K'") ;
- query(eci, "review_profilename = 'Twoapennything'") ;
- query(eci, "review_profilename contains 'Natalia'") ;
- query(eci, "review_profilename contains 'Patrick'") ;
- query(eci, "review_time = 1342051200") ;
- query(eci, "review_time > 1342051200") ;
- query(eci, "review_score > 0");
- query(eci, "review_score > 2");
- query(eci, "review_score > 3");
- query(eci, "review_score > 4");
- query(eci, "review_score > 5");
+ query(indexScope, eci, "product_productid = 'B006K2ZZ7K'") ;
+ query(indexScope, eci, "review_profilename = 'Twoapennything'") ;
+ query(indexScope, eci, "review_profilename contains 'Natalia'") ;
+ query(indexScope, eci, "review_profilename contains 'Patrick'") ;
+ query(indexScope, eci, "review_time = 1342051200") ;
+ query(indexScope, eci, "review_time > 1342051200") ;
+ query(indexScope, eci, "review_score > 0");
+ query(indexScope, eci, "review_score > 2");
+ query(indexScope, eci, "review_score > 3");
+ query(indexScope, eci, "review_score > 4");
+ query(indexScope, eci, "review_score > 5");
}
}
- public static void query( EntityIndex eci, String query ) {;
+ public static void query(final IndexScope indexScope, final EntityIndex eci, final String query ) {;
Query q = Query.fromQL(query) ;
- CandidateResults candidateResults = eci.search( q );
+ CandidateResults candidateResults = eci.search(indexScope, q );
log.info("size = {} returned from query {}", candidateResults.size(), q.getQl() );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fba71c2d/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java
index f8df2a2..5fb02f2 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java
@@ -36,6 +36,7 @@ import org.apache.usergrid.persistence.collection.util.EntityUtils;
import org.apache.usergrid.persistence.core.cassandra.CassandraRule;
import org.apache.usergrid.persistence.core.cassandra.ITRunner;
import org.apache.usergrid.persistence.index.EntityIndex;
+import org.apache.usergrid.persistence.index.EntityIndexBatch;
import org.apache.usergrid.persistence.index.EntityIndexFactory;
import org.apache.usergrid.persistence.index.IndexScope;
import org.apache.usergrid.persistence.index.guice.TestIndexModule;
@@ -105,12 +106,14 @@ public class EntityConnectionIndexImplTest extends BaseIT {
IndexScope scope = new IndexScopeImpl( appId, person.getId(), "likes" );
EntityIndex personLikesIndex = ecif.createEntityIndex( scope );
- personLikesIndex.index( muffin );
- personLikesIndex.refresh();
+ EntityIndexBatch batch = personLikesIndex.createBatch();
+
+ batch.index( scope, muffin );
+ batch.executeAndRefresh();
// now, let's search for things that Dave likes
- CandidateResults likes = personLikesIndex.search( Query.fromQL( "select *" ) );
+ CandidateResults likes = personLikesIndex.search(scope, Query.fromQL( "select *" ) );
assertEquals( 1, likes.size() );
assertEquals(muffin.getId(), likes.get(0).getId());
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fba71c2d/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
index d4b8d5b..e01c45d 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
@@ -42,6 +42,7 @@ import org.apache.usergrid.persistence.collection.util.EntityUtils;
import org.apache.usergrid.persistence.core.cassandra.CassandraRule;
import org.apache.usergrid.persistence.core.cassandra.ITRunner;
import org.apache.usergrid.persistence.index.EntityIndex;
+import org.apache.usergrid.persistence.index.EntityIndexBatch;
import org.apache.usergrid.persistence.index.EntityIndexFactory;
import org.apache.usergrid.persistence.index.IndexScope;
import org.apache.usergrid.persistence.index.guice.TestIndexModule;
@@ -104,6 +105,9 @@ public class EntityIndexTest extends BaseIT {
int count = 0;
StopWatch timer = new StopWatch();
timer.start();
+
+ final EntityIndexBatch batch = entityIndex.createBatch();
+
for ( Object o : sampleJson ) {
Map<String, Object> item = ( Map<String, Object> ) o;
@@ -112,12 +116,20 @@ public class EntityIndexTest extends BaseIT {
entity = EntityIndexMapUtils.fromMap( entity, item );
EntityUtils.setVersion( entity, UUIDGenerator.newTimeUUID() );
- entityIndex.index( entity );
+ batch.index( indexScope, entity );
+
+ if(count %1000 == 0){
+ batch.execute();
+ }
+
+
if ( count++ > MAX_ENTITIES ) {
break;
}
}
+
+ batch.execute();
timer.stop();
log.info( "Total time to index {} entries {}ms, average {}ms/entry",
new Object[] { count, timer.getTime(), timer.getTime() / count } );
@@ -125,7 +137,7 @@ public class EntityIndexTest extends BaseIT {
entityIndex.refresh();
- testQueries( entityIndex );
+ testQueries( indexScope, entityIndex );
}
@@ -147,29 +159,27 @@ public class EntityIndexTest extends BaseIT {
Entity entity = EntityIndexMapUtils.fromMap( entityMap );
EntityUtils.setId( entity, new SimpleId( "fastcar" ) );
EntityUtils.setVersion( entity, UUIDGenerator.newTimeUUID() );
- entityIndex.index( entity );
+ entityIndex.createBatch().index(indexScope , entity ).executeAndRefresh();
- entityIndex.refresh();
-
- CandidateResults candidateResults = entityIndex.search( Query.fromQL( "name contains 'Ferrari*'" ) );
+ CandidateResults candidateResults = entityIndex.search(indexScope, Query.fromQL( "name contains 'Ferrari*'" ) );
assertEquals( 1, candidateResults.size() );
- entityIndex.deindex( entity );
+ entityIndex.createBatch().deindex( indexScope, entity ).execute();
entityIndex.refresh();
- candidateResults = entityIndex.search( Query.fromQL( "name contains 'Ferrari*'" ) );
+ candidateResults = entityIndex.search( indexScope, Query.fromQL( "name contains 'Ferrari*'" ) );
assertEquals( 0, candidateResults.size() );
}
- private void testQuery( EntityIndex entityIndex, String queryString, int num ) {
+ private void testQuery(final IndexScope scope, final EntityIndex entityIndex, final String queryString, final int num ) {
StopWatch timer = new StopWatch();
timer.start();
Query query = Query.fromQL( queryString );
query.setLimit( 1000 );
- CandidateResults candidateResults = entityIndex.search( query );
+ CandidateResults candidateResults = entityIndex.search( scope, query );
timer.stop();
assertEquals( num, candidateResults.size() );
@@ -177,34 +187,34 @@ public class EntityIndexTest extends BaseIT {
}
- private void testQueries( EntityIndex entityIndex ) {
+ private void testQueries(final IndexScope scope, final EntityIndex entityIndex ) {
- testQuery( entityIndex, "name = 'Morgan Pierce'", 1 );
+ testQuery(scope, entityIndex, "name = 'Morgan Pierce'", 1 );
- testQuery( entityIndex, "name = 'morgan pierce'", 1 );
+ testQuery(scope, entityIndex, "name = 'morgan pierce'", 1 );
- testQuery( entityIndex, "name = 'Morgan'", 0 );
+ testQuery(scope, entityIndex, "name = 'Morgan'", 0 );
- testQuery( entityIndex, "name contains 'Morgan'", 1 );
+ testQuery(scope, entityIndex, "name contains 'Morgan'", 1 );
- testQuery( entityIndex, "company > 'GeoLogix'", 64 );
+ testQuery(scope, entityIndex, "company > 'GeoLogix'", 64 );
- testQuery( entityIndex, "gender = 'female'", 45 );
+ testQuery(scope, entityIndex, "gender = 'female'", 45 );
- testQuery( entityIndex, "name = 'Minerva Harrell' and age > 39", 1 );
+ testQuery(scope, entityIndex, "name = 'Minerva Harrell' and age > 39", 1 );
- testQuery( entityIndex, "name = 'Minerva Harrell' and age > 39 and age < 41", 1 );
+ testQuery(scope, entityIndex, "name = 'Minerva Harrell' and age > 39 and age < 41", 1 );
- testQuery( entityIndex, "name = 'Minerva Harrell' and age > 40", 0 );
+ testQuery(scope, entityIndex, "name = 'Minerva Harrell' and age > 40", 0 );
- testQuery( entityIndex, "name = 'Minerva Harrell' and age >= 40", 1 );
+ testQuery(scope, entityIndex, "name = 'Minerva Harrell' and age >= 40", 1 );
- testQuery( entityIndex, "name = 'Minerva Harrell' and age <= 40", 1 );
+ testQuery(scope, entityIndex, "name = 'Minerva Harrell' and age <= 40", 1 );
- testQuery( entityIndex, "name = 'Morgan* '", 1 );
+ testQuery(scope, entityIndex, "name = 'Morgan* '", 1 );
- testQuery( entityIndex, "name = 'Morgan*'", 1 );
+ testQuery(scope, entityIndex, "name = 'Morgan*'", 1 );
// test a couple of array sub-property queries
@@ -212,16 +222,16 @@ public class EntityIndexTest extends BaseIT {
int totalUsers = 102;
// nobody has a friend named Jack the Ripper
- testQuery( entityIndex, "friends.name = 'Jack the Ripper'", 0 );
+ testQuery(scope, entityIndex, "friends.name = 'Jack the Ripper'", 0 );
// everybody doesn't have a friend named Jack the Ripper
- testQuery( entityIndex, "not (friends.name = 'Jack the Ripper')", totalUsers );
+ testQuery(scope, entityIndex, "not (friends.name = 'Jack the Ripper')", totalUsers );
// one person has a friend named Shari Hahn
- testQuery( entityIndex, "friends.name = 'Wendy Moody'", 1 );
+ testQuery(scope, entityIndex, "friends.name = 'Wendy Moody'", 1 );
// everybody but 1 doesn't have a friend named Shari Hahh
- testQuery( entityIndex, "not (friends.name = 'Shari Hahn')", totalUsers - 1);
+ testQuery(scope, entityIndex, "not (friends.name = 'Shari Hahn')", totalUsers - 1);
}
@@ -282,17 +292,20 @@ public class EntityIndexTest extends BaseIT {
EntityUtils.setId( user, new SimpleId( "edanuff" ) );
EntityUtils.setVersion( user, UUIDGenerator.newTimeUUID() );
- entityIndex.index( user );
+
+ final EntityIndexBatch batch = entityIndex.createBatch();
+
+ batch.index( indexScope, user );
user.setField( new StringField( "address1", "1782 address st" ) );
- entityIndex.index( user );
+ batch.index( indexScope, user );
user.setField( new StringField( "address2", "apt 508" ) );
- entityIndex.index( user );
+ batch.index( indexScope, user );
user.setField( new StringField( "address3", "apt 508" ) );
- entityIndex.index( user );
- entityIndex.refresh();
+ batch.index( indexScope, user );
+ batch.executeAndRefresh();
- CandidateResults results = entityIndex.getEntityVersions( user.getId() );
+ CandidateResults results = entityIndex.getEntityVersions(indexScope, user.getId() );
assertEquals(1, results.size());
assertEquals( results.get( 0 ).getId(), user.getId() );
@@ -323,21 +336,21 @@ public class EntityIndexTest extends BaseIT {
EntityUtils.setVersion( user, UUIDGenerator.newTimeUUID() );
- ei.index( user );
- ei.refresh();
+ EntityIndexBatch batch = ei.createBatch();
+
+ batch.index( appScope, user ).executeAndRefresh();
Query query = new Query();
query.addEqualityFilter( "username", "edanuff" );
- CandidateResults r = ei.search( query );
+ CandidateResults r = ei.search( appScope, query );
assertEquals( user.getId(), r.get( 0 ).getId() );
- ei.deindex( user.getId(), user.getVersion() );
- ei.refresh();
+ batch.deindex(appScope, user.getId(), user.getVersion() ).executeAndRefresh();
// EntityRef
query = new Query();
query.addEqualityFilter( "username", "edanuff" );
- r = ei.search( query );
+ r = ei.search(appScope, query );
assertFalse( r.iterator().hasNext() );
}
@@ -362,7 +375,10 @@ public class EntityIndexTest extends BaseIT {
Entity bill = EntityIndexMapUtils.fromMap( billMap );
EntityUtils.setId( bill, new SimpleId( UUIDGenerator.newTimeUUID(), "user" ) );
EntityUtils.setVersion( bill, UUIDGenerator.newTimeUUID() );
- ei.index( bill );
+
+ EntityIndexBatch batch = ei.createBatch();
+
+ batch.index( appScope, bill );
// Fred has age as int, favorites as object and retirement goal as object
Map fredMap = new HashMap() {{
@@ -382,28 +398,28 @@ public class EntityIndexTest extends BaseIT {
Entity fred = EntityIndexMapUtils.fromMap( fredMap );
EntityUtils.setId( fred, new SimpleId( UUIDGenerator.newTimeUUID(), "user" ) );
EntityUtils.setVersion( fred, UUIDGenerator.newTimeUUID() );
- ei.index( fred );
+ batch.index( appScope, fred );
- ei.refresh();
+ batch.executeAndRefresh();
Query query = new Query();
query.addEqualityFilter( "username", "bill" );
- CandidateResults r = ei.search( query );
+ CandidateResults r = ei.search( appScope, query );
assertEquals( bill.getId(), r.get( 0 ).getId() );
query = new Query();
query.addEqualityFilter( "username", "fred" );
- r = ei.search( query );
+ r = ei.search( appScope, query );
assertEquals( fred.getId(), r.get( 0 ).getId() );
query = new Query();
query.addEqualityFilter( "age", 41 );
- r = ei.search( query );
+ r = ei.search( appScope, query );
assertEquals( fred.getId(), r.get( 0 ).getId() );
query = new Query();
query.addEqualityFilter( "age", "thirtysomething" );
- r = ei.search( query );
+ r = ei.search( appScope, query );
assertEquals( bill.getId(), r.get( 0 ).getId() );
}
}
[11/13] git commit: Updated naming scheme so that we can efficiently
seek meta data by connection or collection suffix in graph
Posted by to...@apache.org.
Updated naming scheme so that we can efficiently seek meta data by connection or collection suffix in graph
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/0e18f57f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/0e18f57f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/0e18f57f
Branch: refs/heads/two-dot-o
Commit: 0e18f57f81c2bfdf1dccf9e0b2c19ea6c581ecdd
Parents: 93471f7
Author: Todd Nine <to...@apache.org>
Authored: Thu Oct 2 00:36:25 2014 -0600
Committer: Todd Nine <to...@apache.org>
Committed: Thu Oct 2 00:36:25 2014 -0600
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 140 ++++---------------
.../corepersistence/CpEntityManagerFactory.java | 5 +-
.../usergrid/corepersistence/CpNamingUtils.java | 106 ++++++++++++++
.../corepersistence/CpRelationManager.java | 97 ++++++-------
4 files changed, 184 insertions(+), 164 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0e18f57f/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 320d3fe..777b041 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -53,9 +53,6 @@ import me.prettyprint.hector.api.query.QueryResult;
import me.prettyprint.hector.api.query.SliceCounterQuery;
import static org.apache.commons.lang.StringUtils.capitalize;
import static org.apache.commons.lang.StringUtils.isBlank;
-import static org.apache.usergrid.corepersistence.CpRelationManager.ALL_TYPES;
-import static org.apache.usergrid.corepersistence.CpRelationManager.EDGE_COLL_SUFFIX;
-import static org.apache.usergrid.corepersistence.CpRelationManager.EDGE_CONN_SUFFIX;
import org.apache.usergrid.persistence.AggregateCounter;
import org.apache.usergrid.persistence.AggregateCounterSet;
import org.apache.usergrid.persistence.CollectionRef;
@@ -167,8 +164,6 @@ public class CpEntityManager implements EntityManager {
public static final String APPLICATION_COLLECTION = "application.collection.";
public static final String APPLICATION_ENTITIES = "application.entities";
public static final long ONE_COUNT = 1L;
- private static final String COLL_SUFFIX = "zzzcollzzz";
- private static final String CONN_SUFFIX = "zzzconnzzz";
private UUID applicationId;
private Application application;
@@ -214,82 +209,6 @@ public class CpEntityManager implements EntityManager {
}
- static String getCollectionScopeNameFromEntityType( String type ) {
- String csn = Schema.defaultCollectionName( type ) + COLL_SUFFIX;
- return csn.toLowerCase();
- }
-
-
- static String getCollectionScopeNameFromCollectionName( String name ) {
- String csn = name + COLL_SUFFIX;
- return csn.toLowerCase();
- }
-
-
- static String getConnectionScopeName( String entityType, String connectionType ) {
- String csn = connectionType + entityType + CONN_SUFFIX;
- return csn.toLowerCase();
- }
-
- static boolean isCollectionEdgeType( String type ) {
- return type.endsWith( EDGE_COLL_SUFFIX );
- }
-
- static boolean isConnectionEdgeType( String type ) {
- return type.endsWith( EDGE_CONN_SUFFIX );
- }
-
- static public String getConnectionType( String edgeType ) {
- String[] parts = edgeType.split("\\|");
- return parts[0];
- }
-
- static public String getCollectionEntityType( String edgeType ) {
- String[] parts = edgeType.split("\\|");
- return parts[1];
- }
-
- static public String getCollectionName( String edgeType ) {
- String[] parts = edgeType.split("\\|");
- return parts[0];
- }
-
- static public String getConnectedEntityType( String edgeType ) {
- String[] parts = edgeType.split("\\|");
- return parts[1];
- }
-
- static String getEdgeTypeFromConnectionType( String connectionType, String targetEntityType ) {
-
- if ( connectionType != null && targetEntityType != null ) {
- String csn = connectionType + "|" + targetEntityType + "|" + EDGE_CONN_SUFFIX;
- return csn;
- }
-
- if ( connectionType != null ) {
- // no suffix, this must be a search
- String csn = connectionType;
- return csn;
- }
-
- return null;
- }
-
- static String getEdgeTypeFromCollectionName( String collectionName, String targetEntityType ) {
-
- if ( collectionName != null && targetEntityType != null ) {
- String csn = collectionName + "|" + targetEntityType + "|" + EDGE_COLL_SUFFIX;
- return csn;
- }
-
- if ( collectionName != null ) {
- // no suffix, this must be a search
- String csn = collectionName;
- return csn;
- }
-
- return null;
- }
@Override
@@ -380,7 +299,7 @@ public class CpEntityManager implements EntityManager {
}
Id id = new SimpleId( entityRef.getUuid(), entityRef.getType() );
- String collectionName = getCollectionScopeNameFromEntityType( entityRef.getType() );
+ String collectionName = CpNamingUtils.getCollectionScopeNameFromEntityType( entityRef.getType() );
CollectionScope collectionScope = new CollectionScopeImpl(
applicationScope.getApplication(), applicationScope.getApplication(), collectionName );
@@ -467,7 +386,7 @@ public class CpEntityManager implements EntityManager {
String type = Schema.getDefaultSchema().getEntityType( entityClass );
Id id = new SimpleId( entityId, type );
- String collectionName = getCollectionScopeNameFromEntityType( type );
+ String collectionName = CpNamingUtils.getCollectionScopeNameFromEntityType( type );
CollectionScope collectionScope = new CollectionScopeImpl(
applicationScope.getApplication(), applicationScope.getApplication(), collectionName );
@@ -538,7 +457,7 @@ public class CpEntityManager implements EntityManager {
CollectionScope collectionScope = new CollectionScopeImpl(
applicationScope.getApplication(),
applicationScope.getApplication(),
- getCollectionScopeNameFromEntityType( entity.getType() ) );
+ CpNamingUtils.getCollectionScopeNameFromEntityType( entity.getType() ) );
EntityCollectionManager ecm = managerCache.getEntityCollectionManager( collectionScope );
Id entityId = new SimpleId( entity.getUuid(), entity.getType() );
@@ -558,8 +477,12 @@ public class CpEntityManager implements EntityManager {
// "Entity Id " + entityId.getType() + ":"+ entityId.getUuid() +" uuid not time based");
// }
- org.apache.usergrid.persistence.model.entity.Entity cpEntity =
- ecm.load( entityId ).toBlockingObservable().last();
+// org.apache.usergrid.persistence.model.entity.Entity cpEntity =
+// ecm.load( entityId ).toBlockingObservable().last();
+
+
+ org.apache.usergrid.persistence.model.entity.Entity cpEntity = new org.apache.usergrid.persistence.model
+ .entity.Entity( entityId );
cpEntity = CpEntityMapUtils.fromMap(
cpEntity, entity.getProperties(), entity.getType(), true );
@@ -569,7 +492,6 @@ public class CpEntityManager implements EntityManager {
cpEntity.getId().getType(), cpEntity.getId().getUuid(), cpEntity.getVersion() });
cpEntity = ecm.update( cpEntity ).toBlockingObservable().last();
- refreshIndex();
cpEntity = ecm.load( entityId ).toBlockingObservable().last();
logger.debug("Wrote {}:{} version {}", new Object[] {
@@ -603,7 +525,7 @@ public class CpEntityManager implements EntityManager {
CollectionScope collectionScope = new CollectionScopeImpl(
applicationScope.getApplication(),
applicationScope.getApplication(),
- getCollectionScopeNameFromEntityType( entityRef.getType() ) );
+ CpNamingUtils.getCollectionScopeNameFromEntityType( entityRef.getType() ) );
EntityCollectionManager ecm = managerCache.getEntityCollectionManager( collectionScope );
@@ -643,7 +565,7 @@ public class CpEntityManager implements EntityManager {
IndexScope indexScope = new IndexScopeImpl(
applicationScope.getApplication(),
new SimpleId( uuid, ownerType ),
- CpEntityManager.getCollectionScopeNameFromCollectionName(coll) );
+ CpNamingUtils.getCollectionScopeNameFromCollectionName( coll ) );
batch.index( indexScope, entity );
@@ -657,14 +579,14 @@ public class CpEntityManager implements EntityManager {
IndexScope defaultIndexScope = new IndexScopeImpl(
applicationScope.getApplication(),
applicationScope.getApplication(),
- getCollectionScopeNameFromEntityType( entityRef.getType() ) );
+ CpNamingUtils.getCollectionScopeNameFromEntityType( entityRef.getType() ) );
batch.deindex(defaultIndexScope, entity );
IndexScope allTypesIndexScope = new IndexScopeImpl(
applicationScope.getApplication(),
applicationScope.getApplication(),
- ALL_TYPES);
+ CpNamingUtils.ALL_TYPES);
batch.deindex( allTypesIndexScope, entity );
@@ -1042,7 +964,7 @@ public class CpEntityManager implements EntityManager {
@Override
public void deleteProperty( EntityRef entityRef, String propertyName ) throws Exception {
- String collectionName = getCollectionScopeNameFromEntityType( entityRef.getType() );
+ String collectionName = CpNamingUtils.getCollectionScopeNameFromEntityType( entityRef.getType() );
CollectionScope collectionScope = new CollectionScopeImpl(
applicationScope.getApplication(),
@@ -1052,7 +974,7 @@ public class CpEntityManager implements EntityManager {
IndexScope defaultIndexScope = new IndexScopeImpl(
applicationScope.getApplication(),
applicationScope.getApplication(),
- getCollectionScopeNameFromEntityType( entityRef.getType()) );
+ CpNamingUtils.getCollectionScopeNameFromEntityType( entityRef.getType() ) );
EntityCollectionManager ecm = managerCache.getEntityCollectionManager( collectionScope );
EntityIndex ei = managerCache.getEntityIndex( applicationScope );
@@ -2570,7 +2492,7 @@ public class CpEntityManager implements EntityManager {
CollectionScope collectionScope = new CollectionScopeImpl(
applicationScope.getApplication(),
applicationScope.getApplication(),
- getCollectionScopeNameFromEntityType( eType ) );
+ CpNamingUtils.getCollectionScopeNameFromEntityType( eType ) );
EntityCollectionManager ecm = managerCache.getEntityCollectionManager( collectionScope );
if ( logger.isDebugEnabled() ) {
@@ -2911,15 +2833,15 @@ public class CpEntityManager implements EntityManager {
@Override
public void call( Edge edge ) {
- if ( isCollectionEdgeType( edge.getType() )) {
+ if ( CpNamingUtils.isCollectionEdgeType( edge.getType() )) {
- String collName = getCollectionName(edgeType);
- String memberType = getCollectionEntityType(edgeType);
+ String collName = CpNamingUtils.getCollectionName( edgeType );
+ String memberType = edge.getTargetNode().getType();
CollectionScope collScope = new CollectionScopeImpl(
applicationScope.getApplication(),
applicationScope.getApplication(),
- CpEntityManager.getCollectionScopeNameFromEntityType(entity.getType()));
+ CpNamingUtils.getCollectionScopeNameFromEntityType( entity.getType() ));
EntityCollectionManager collMgr =
managerCache.getEntityCollectionManager(collScope);
@@ -2944,7 +2866,7 @@ public class CpEntityManager implements EntityManager {
CollectionScope memberScope = new CollectionScopeImpl(
applicationScope.getApplication(),
applicationScope.getApplication(),
- CpEntityManager.getCollectionScopeNameFromEntityType(memberType));
+ CpNamingUtils.getCollectionScopeNameFromEntityType( memberType ));
EntityCollectionManager memberMgr =
managerCache.getEntityCollectionManager(memberScope);
@@ -2976,16 +2898,16 @@ public class CpEntityManager implements EntityManager {
indexEntityConnectionsAndCollections( new SimpleEntityRef(
memberEntity.getId().getType(), memberEntity.getId().getUuid()),po);
- } else if ( isConnectionEdgeType( edge.getType() )) {
+ } else if ( CpNamingUtils.isConnectionEdgeType( edge.getType() )) {
- String connType = getConnectionType(edgeType);
- String targetEntityType = getConnectedEntityType(edgeType);
+ String connType = CpNamingUtils.getConnectionType( edgeType );
+ String targetEntityType = edge.getTargetNode().getType();
String sourceEntityType = entity.getType();
CollectionScope sourceScope = new CollectionScopeImpl(
applicationScope.getApplication(),
applicationScope.getApplication(),
- CpEntityManager.getCollectionScopeNameFromEntityType(sourceEntityType));
+ CpNamingUtils.getCollectionScopeNameFromEntityType( sourceEntityType ));
EntityCollectionManager sourceEcm =
managerCache.getEntityCollectionManager(sourceScope);
@@ -2995,7 +2917,7 @@ public class CpEntityManager implements EntityManager {
CollectionScope targetScope = new CollectionScopeImpl(
applicationScope.getApplication(),
applicationScope.getApplication(),
- CpEntityManager.getCollectionScopeNameFromEntityType(targetEntityType));
+ CpNamingUtils.getCollectionScopeNameFromEntityType( targetEntityType ));
EntityCollectionManager targetEcm =
managerCache.getEntityCollectionManager(targetScope);
@@ -3071,14 +2993,14 @@ public class CpEntityManager implements EntityManager {
IndexScope indexScope = new IndexScopeImpl(
applicationScope.getApplication(),
sourceEntity.getId(),
- CpEntityManager.getConnectionScopeName(targetEntityType, connType));
+ CpNamingUtils.getConnectionScopeName( targetEntityType, connType ));
batch.index(indexScope, targetEntity);
// Index the new connection in app|scope|all-types context
IndexScope allTypesIndexScope = new IndexScopeImpl(
applicationScope.getApplication(),
sourceEntity.getId(),
- ALL_TYPES);
+ CpNamingUtils.ALL_TYPES);
batch.index(allTypesIndexScope, targetEntity);
@@ -3098,7 +3020,7 @@ public class CpEntityManager implements EntityManager {
IndexScope collectionIndexScope = new IndexScopeImpl(
applicationScope.getApplication(),
collectionEntity.getId(),
- CpEntityManager.getCollectionScopeNameFromCollectionName(collName));
+ CpNamingUtils.getCollectionScopeNameFromCollectionName( collName ));
batch.index(collectionIndexScope, memberEntity);
@@ -3106,7 +3028,7 @@ public class CpEntityManager implements EntityManager {
IndexScope entityAllTypesScope = new IndexScopeImpl(
applicationScope.getApplication(),
collectionEntity.getId(),
- ALL_TYPES);
+ CpNamingUtils.ALL_TYPES);
batch.index(entityAllTypesScope, memberEntity);
@@ -3114,7 +3036,7 @@ public class CpEntityManager implements EntityManager {
IndexScope appAllTypesScope = new IndexScopeImpl(
applicationScope.getApplication(),
applicationScope.getApplication(),
- ALL_TYPES);
+ CpNamingUtils.ALL_TYPES);
batch.index(appAllTypesScope, memberEntity);
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0e18f57f/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index 9a10afd..f24f26b 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -51,7 +51,6 @@ import org.apache.usergrid.persistence.graph.GraphManager;
import org.apache.usergrid.persistence.graph.GraphManagerFactory;
import org.apache.usergrid.persistence.graph.SearchByEdgeType;
import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdgeType;
-import org.apache.usergrid.persistence.index.EntityIndex;
import org.apache.usergrid.persistence.index.EntityIndexFactory;
import org.apache.usergrid.persistence.index.IndexScope;
import org.apache.usergrid.persistence.index.impl.IndexScopeImpl;
@@ -349,7 +348,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
Application app = em.getApplication();
Id fromEntityId = new SimpleId( app.getUuid(), app.getType() );
- String edgeType = CpEntityManager.getEdgeTypeFromCollectionName("appinfos", "appinfo");
+ String edgeType = CpNamingUtils.getEdgeTypeFromCollectionName( "appinfos" );
logger.debug("getApplications(): Loading edges of edgeType {} from {}:{}",
new Object[] { edgeType, fromEntityId.getType(), fromEntityId.getUuid() } );
@@ -372,7 +371,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
CollectionScope collScope = new CollectionScopeImpl(
appScope.getApplication(),
appScope.getApplication(),
- CpEntityManager.getCollectionScopeNameFromCollectionName("appinfos"));
+ CpNamingUtils.getCollectionScopeNameFromCollectionName( "appinfos" ));
org.apache.usergrid.persistence.model.entity.Entity e =
managerCache.getEntityCollectionManager( collScope ).load( targetId )
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0e18f57f/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpNamingUtils.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpNamingUtils.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpNamingUtils.java
new file mode 100644
index 0000000..60cec72
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpNamingUtils.java
@@ -0,0 +1,106 @@
+package org.apache.usergrid.corepersistence;/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+import org.apache.usergrid.persistence.Schema;
+
+
+/**
+ * Utilises for constructing standard naming conventions for collections and connections
+ */
+public class CpNamingUtils {
+
+ /**
+ * Edge types for all types
+ */
+ static final String ALL_TYPES = "zzzalltypeszzz";
+
+ /**
+ * Edge types for collection suffix
+ */
+ static final String EDGE_COLL_SUFFIX = "zzzcollzzz";
+
+ /**
+ * Edge types for connection suffix
+ */
+ static final String EDGE_CONN_SUFFIX = "zzzconnzzz";
+
+ static String getCollectionScopeNameFromEntityType( String type ) {
+ String csn = EDGE_COLL_SUFFIX + Schema.defaultCollectionName( type );
+ return csn.toLowerCase();
+ }
+
+
+ static String getCollectionScopeNameFromCollectionName( String name ) {
+ String csn = EDGE_COLL_SUFFIX + name ;
+ return csn.toLowerCase();
+ }
+
+
+ static String getConnectionScopeName( String entityType, String connectionType ) {
+ String csn = EDGE_CONN_SUFFIX + connectionType + entityType ;
+ return csn.toLowerCase();
+ }
+
+ static boolean isCollectionEdgeType( String type ) {
+ return type.startsWith( EDGE_COLL_SUFFIX );
+ }
+
+ static boolean isConnectionEdgeType( String type ) {
+ return type.startsWith( EDGE_CONN_SUFFIX );
+ }
+
+
+
+ static public String getConnectionType( String edgeType ) {
+ String[] parts = edgeType.split("\\|");
+ return parts[1];
+ }
+
+
+ static public String getCollectionName( String edgeType ) {
+ String[] parts = edgeType.split("\\|");
+ return parts[1];
+ }
+
+
+
+ static String getEdgeTypeFromConnectionType( String connectionType) {
+
+ if ( connectionType != null ) {
+ String csn = EDGE_CONN_SUFFIX + "|" + connectionType;
+ return csn;
+ }
+
+ return null;
+ }
+
+ static String getEdgeTypeFromCollectionName( String collectionName) {
+
+ if ( collectionName != null ) {
+ String csn = EDGE_COLL_SUFFIX + "|" + collectionName;
+ return csn;
+ }
+
+
+
+ return null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0e18f57f/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index 14f9359..ca01ddd 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -17,7 +17,7 @@
package org.apache.usergrid.corepersistence;
import static me.prettyprint.hector.api.factory.HFactory.createMutator;
-import static org.apache.usergrid.corepersistence.CpEntityManager.*;
+
import com.yammer.metrics.annotation.Metered;
import java.nio.ByteBuffer;
import java.util.AbstractMap;
@@ -138,11 +138,7 @@ import org.apache.usergrid.utils.IndexUtils;
import static org.apache.usergrid.utils.InflectionUtils.singularize;
import org.apache.usergrid.utils.MapUtils;
import static org.apache.usergrid.utils.MapUtils.addMapSet;
-import org.apache.usergrid.utils.UUIDUtils;
import static org.apache.usergrid.utils.UUIDUtils.getTimestampInMicros;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.util.Assert;
import me.prettyprint.hector.api.Keyspace;
import me.prettyprint.hector.api.beans.DynamicComposite;
@@ -158,11 +154,7 @@ public class CpRelationManager implements RelationManager {
private static final Logger logger = LoggerFactory.getLogger( CpRelationManager.class );
- static final String ALL_TYPES = "zzzalltypeszzz";
- static final String EDGE_COLL_SUFFIX = "zzzcollzzz";
-
- static final String EDGE_CONN_SUFFIX = "zzzconnzzz";
private CpEntityManagerFactory emf;
@@ -218,7 +210,7 @@ public class CpRelationManager implements RelationManager {
this.headEntityScope = new CollectionScopeImpl(
this.applicationScope.getApplication(),
this.applicationScope.getApplication(),
- CpEntityManager.getCollectionScopeNameFromEntityType( headEntity.getType()));
+ CpNamingUtils.getCollectionScopeNameFromEntityType( headEntity.getType() ));
EntityCollectionManager ecm = managerCache.getEntityCollectionManager(headEntityScope);
if ( logger.isDebugEnabled() ) {
@@ -248,7 +240,7 @@ public class CpRelationManager implements RelationManager {
GraphManager gm = managerCache.getGraphManager(applicationScope);
- String edgeTypePrefix = getEdgeTypeFromCollectionName( collectionName, null );
+ String edgeTypePrefix = CpNamingUtils.getEdgeTypeFromCollectionName( collectionName );
logger.debug("getCollectionIndexes(): Searching for edge type prefix {} to target {}:{}",
new Object[] {
@@ -337,10 +329,10 @@ public class CpRelationManager implements RelationManager {
edge.getSourceNode().getType(), edge.getSourceNode().getUuid() );
String name = null;
- if ( CpEntityManager.isConnectionEdgeType( edge.getType() )) {
- name = CpEntityManager.getConnectionType( edge.getType() );
+ if ( CpNamingUtils.isConnectionEdgeType( edge.getType() )) {
+ name = CpNamingUtils.getConnectionType( edge.getType() );
} else {
- name = CpEntityManager.getCollectionName( edge.getType() );
+ name = CpNamingUtils.getCollectionName( edge.getType() );
}
addMapSet( results, eref, name );
}
@@ -400,21 +392,21 @@ public class CpRelationManager implements RelationManager {
// reindex the entity in the source entity's collection or connection index
IndexScope indexScope;
- if ( CpEntityManager.isCollectionEdgeType( edge.getType() )) {
+ if ( CpNamingUtils.isCollectionEdgeType( edge.getType() )) {
- String collName = CpEntityManager.getCollectionName( edge.getType() );
+ String collName = CpNamingUtils.getCollectionName( edge.getType() );
indexScope = new IndexScopeImpl(
applicationScope.getApplication(),
new SimpleId(sourceEntity.getUuid(), sourceEntity.getType()),
- CpEntityManager.getCollectionScopeNameFromCollectionName( collName ));
+ CpNamingUtils.getCollectionScopeNameFromCollectionName( collName ));
} else {
- String connName = CpEntityManager.getCollectionName( edge.getType() );
+ String connName = CpNamingUtils.getCollectionName( edge.getType() );
indexScope = new IndexScopeImpl(
applicationScope.getApplication(),
new SimpleId(sourceEntity.getUuid(), sourceEntity.getType()),
- CpEntityManager.getConnectionScopeName( cpHeadEntity.getId().getType(), connName ));
+ CpNamingUtils.getConnectionScopeName( cpHeadEntity.getId().getType(), connName ));
}
entityIndexBatch.index(indexScope, cpEntity);
@@ -424,13 +416,16 @@ public class CpRelationManager implements RelationManager {
indexScope = new IndexScopeImpl(
applicationScope.getApplication(),
new SimpleId(sourceEntity.getUuid(), sourceEntity.getType()),
- ALL_TYPES);
+ CpNamingUtils.ALL_TYPES);
entityIndexBatch.index(indexScope, cpEntity);
count++;
}
}
+
+ entityIndexBatch.execute();
+
logger.debug("updateContainingCollectionsAndCollections() updated {} indexes", count);
return results;
}
@@ -441,7 +436,7 @@ public class CpRelationManager implements RelationManager {
Id entityId = new SimpleId( entity.getUuid(), entity.getType() );
- String edgeType = getEdgeTypeFromConnectionType( connectionType, entity.getType() );
+ String edgeType = CpNamingUtils.getEdgeTypeFromConnectionType( connectionType );
logger.debug("isConnectionMember(): Checking for edge type {} from {}:{} to {}:{}",
new Object[] {
@@ -469,7 +464,7 @@ public class CpRelationManager implements RelationManager {
Id entityId = new SimpleId( entity.getUuid(), entity.getType() );
- String edgeType = getEdgeTypeFromCollectionName( collName, entity.getType() );
+ String edgeType = CpNamingUtils.getEdgeTypeFromCollectionName( collName );
logger.debug("isCollectionMember(): Checking for edge type {} from {}:{} to {}:{}",
new Object[] {
@@ -499,7 +494,7 @@ public class CpRelationManager implements RelationManager {
Observable<Edge> edgesToTarget = gm.loadEdgesToTarget( new SimpleSearchByEdgeType(
targetId,
- CpEntityManager.getEdgeTypeFromConnectionType( connectionType, target.getType() ),
+ CpNamingUtils.getEdgeTypeFromConnectionType( connectionType ),
System.currentTimeMillis(), SearchByEdgeType.Order.DESCENDING,
null)); // last
@@ -523,7 +518,7 @@ public class CpRelationManager implements RelationManager {
Observable<Edge> edgesFromSource = gm.loadEdgesFromSource(new SimpleSearchByEdgeType(
sourceId,
- CpEntityManager.getEdgeTypeFromConnectionType( connectionType, null ),
+ CpNamingUtils.getEdgeTypeFromConnectionType( connectionType ),
System.currentTimeMillis(),SearchByEdgeType.Order.DESCENDING,
null)); // last
@@ -546,7 +541,7 @@ public class CpRelationManager implements RelationManager {
Iterator<String> iter = str.toBlockingObservable().getIterator();
while ( iter.hasNext() ) {
String edgeType = iter.next();
- indexes.add( CpEntityManager.getCollectionName( edgeType ) );
+ indexes.add( CpNamingUtils.getCollectionName( edgeType ) );
}
return indexes;
@@ -611,8 +606,8 @@ public class CpRelationManager implements RelationManager {
// load the new member entity to be added to the collection from its default scope
CollectionScope memberScope = new CollectionScopeImpl(
applicationScope.getApplication(),
- applicationScope.getApplication(),
- CpEntityManager.getCollectionScopeNameFromEntityType( itemRef.getType()));
+ applicationScope.getApplication(),
+ CpNamingUtils.getCollectionScopeNameFromEntityType( itemRef.getType() ));
EntityCollectionManager memberMgr = managerCache.getEntityCollectionManager(memberScope);
@@ -637,7 +632,7 @@ public class CpRelationManager implements RelationManager {
});
}
- String edgeType = getEdgeTypeFromCollectionName( collName, memberEntity.getId().getType() );
+ String edgeType = CpNamingUtils.getEdgeTypeFromCollectionName( collName );
UUID timeStampUuid = memberEntity.getId().getUuid() != null
&& UUIDUtils.isTimeBased( memberEntity.getId().getUuid())
@@ -763,7 +758,7 @@ public class CpRelationManager implements RelationManager {
CollectionScope memberScope = new CollectionScopeImpl(
this.applicationScope.getApplication(),
this.applicationScope.getApplication(),
- CpEntityManager.getCollectionScopeNameFromEntityType( itemRef.getType() ));
+ CpNamingUtils.getCollectionScopeNameFromEntityType( itemRef.getType() ));
EntityCollectionManager memberMgr = managerCache.getEntityCollectionManager(memberScope);
if ( logger.isDebugEnabled() ) {
@@ -788,7 +783,7 @@ public class CpRelationManager implements RelationManager {
IndexScope indexScope = new IndexScopeImpl(
applicationScope.getApplication(),
cpHeadEntity.getId(),
- CpEntityManager.getCollectionScopeNameFromCollectionName( collName ));
+ CpNamingUtils.getCollectionScopeNameFromCollectionName( collName ));
batch.deindex(indexScope, memberEntity );
@@ -796,8 +791,8 @@ public class CpRelationManager implements RelationManager {
IndexScope itemScope = new IndexScopeImpl(
applicationScope.getApplication(),
memberEntity.getId(),
- CpEntityManager.getCollectionScopeNameFromCollectionName(
- Schema.defaultCollectionName( cpHeadEntity.getId().getType() )));
+ CpNamingUtils.getCollectionScopeNameFromCollectionName(
+ Schema.defaultCollectionName( cpHeadEntity.getId().getType() ) ));
batch.deindex(itemScope, cpHeadEntity );
@@ -808,7 +803,7 @@ public class CpRelationManager implements RelationManager {
GraphManager gm = managerCache.getGraphManager(applicationScope);
Edge collectionToItemEdge = new SimpleEdge(
cpHeadEntity.getId(),
- getEdgeTypeFromCollectionName( collName, memberEntity.getId().getType() ),
+ CpNamingUtils.getEdgeTypeFromCollectionName( collName),
memberEntity.getId(),
memberEntity.getId().getUuid().timestamp() );
gm.deleteEdge(collectionToItemEdge).toBlockingObservable().last();
@@ -816,9 +811,8 @@ public class CpRelationManager implements RelationManager {
// remove edge from item to collection
Edge itemToCollectionEdge = new SimpleEdge(
memberEntity.getId(),
- getEdgeTypeFromCollectionName(
- Schema.defaultCollectionName( cpHeadEntity.getId().getType() ),
- cpHeadEntity.getId().getType() ),
+ CpNamingUtils
+ .getEdgeTypeFromCollectionName( Schema.defaultCollectionName( cpHeadEntity.getId().getType() )),
cpHeadEntity.getId(),
cpHeadEntity.getId().getUuid().timestamp() );
gm.deleteEdge(itemToCollectionEdge).toBlockingObservable().last();
@@ -900,7 +894,7 @@ public class CpRelationManager implements RelationManager {
IndexScope indexScope = new IndexScopeImpl(
applicationScope.getApplication(),
cpHeadEntity.getId(),
- CpEntityManager.getCollectionScopeNameFromCollectionName( collName ));
+ CpNamingUtils.getCollectionScopeNameFromCollectionName( collName ));
EntityIndex ei = managerCache.getEntityIndex(applicationScope);
@@ -981,7 +975,7 @@ public class CpRelationManager implements RelationManager {
CollectionScope targetScope = new CollectionScopeImpl(
applicationScope.getApplication(),
applicationScope.getApplication(),
- CpEntityManager.getCollectionScopeNameFromEntityType( connectedEntityRef.getType() ));
+ CpNamingUtils.getCollectionScopeNameFromEntityType( connectedEntityRef.getType() ));
EntityCollectionManager targetEcm = managerCache.getEntityCollectionManager(targetScope);
@@ -1005,8 +999,8 @@ public class CpRelationManager implements RelationManager {
new SimpleId( connectedEntityRef.getUuid(), connectedEntityRef.getType() ))
.toBlockingObservable().last();
- String edgeType = CpEntityManager
- .getEdgeTypeFromConnectionType( connectionType, connectedEntityRef.getType() );
+ String edgeType = CpNamingUtils
+ .getEdgeTypeFromConnectionType( connectionType );
// create graph edge connection from head entity to member entity
Edge edge = new SimpleEdge(
@@ -1024,7 +1018,7 @@ public class CpRelationManager implements RelationManager {
IndexScope indexScope = new IndexScopeImpl(
applicationScope.getApplication(),
cpHeadEntity.getId(),
- CpEntityManager.getConnectionScopeName( connectedEntityRef.getType(), connectionType ));
+ CpNamingUtils.getConnectionScopeName( connectedEntityRef.getType(), connectionType ));
batch.index(indexScope, targetEntity );
@@ -1032,7 +1026,7 @@ public class CpRelationManager implements RelationManager {
IndexScope allTypesIndexScope = new IndexScopeImpl(
applicationScope.getApplication(),
cpHeadEntity.getId(),
- ALL_TYPES);
+ CpNamingUtils.ALL_TYPES);
batch.index(allTypesIndexScope, targetEntity );
@@ -1213,7 +1207,7 @@ public class CpRelationManager implements RelationManager {
CollectionScope targetScope = new CollectionScopeImpl(
applicationScope.getApplication(),
applicationScope.getApplication(),
- CpEntityManager.getCollectionScopeNameFromEntityType( connectedEntityRef.getType() ));
+ CpNamingUtils.getCollectionScopeNameFromEntityType( connectedEntityRef.getType() ));
EntityCollectionManager targetEcm = managerCache.getEntityCollectionManager(targetScope);
@@ -1248,14 +1242,14 @@ public class CpRelationManager implements RelationManager {
IndexScope indexScope = new IndexScopeImpl(
applicationScope.getApplication(),
new SimpleId( connectingEntityRef.getUuid(), connectingEntityRef.getType() ),
- CpEntityManager.getConnectionScopeName( targetEntity.getId().getType(), connectionType ));
+ CpNamingUtils.getConnectionScopeName( targetEntity.getId().getType(), connectionType ));
batch.deindex( indexScope , targetEntity );
// Deindex the connection in app|source|type context
IndexScope allTypesIndexScope = new IndexScopeImpl(
applicationScope.getApplication(),
new SimpleId( connectingEntityRef.getUuid(), connectingEntityRef.getType() ),
- ALL_TYPES);
+ CpNamingUtils.ALL_TYPES);
batch.deindex( allTypesIndexScope, targetEntity );
@@ -1306,9 +1300,9 @@ public class CpRelationManager implements RelationManager {
String scopeName = null;
if ( connectedEntityType != null ) {
- scopeName = CpEntityManager.getConnectionScopeName( connectedEntityType, connectionType );
+ scopeName = CpNamingUtils.getConnectionScopeName( connectedEntityType, connectionType );
} else {
- scopeName = ALL_TYPES;
+ scopeName = CpNamingUtils.ALL_TYPES;
}
IndexScope indexScope = new IndexScopeImpl(
@@ -1375,7 +1369,7 @@ public class CpRelationManager implements RelationManager {
// looking for edges to the head entity
String edgeType =
- CpEntityManager.getEdgeTypeFromConnectionType( connType, headEntity.getType() );
+ CpNamingUtils.getEdgeTypeFromConnectionType( connType );
Map<EntityRef, Set<String>> containers =
getContainers( count, edgeType, fromEntityType );
@@ -1419,7 +1413,7 @@ public class CpRelationManager implements RelationManager {
IndexScope indexScope = new IndexScopeImpl(
applicationScope.getApplication(),
cpHeadEntity.getId(),
- ALL_TYPES);
+ CpNamingUtils.ALL_TYPES);
EntityIndex ei = managerCache.getEntityIndex(applicationScope);
@@ -1437,8 +1431,7 @@ public class CpRelationManager implements RelationManager {
IndexScope indexScope = new IndexScopeImpl(
applicationScope.getApplication(),
cpHeadEntity.getId(),
- CpEntityManager.getConnectionScopeName(
- query.getEntityType(), query.getConnectionType() ));
+ CpNamingUtils.getConnectionScopeName( query.getEntityType(), query.getConnectionType() ));
EntityIndex ei = managerCache.getEntityIndex(applicationScope);
logger.debug("Searching connections from the '{}' scope {}:{}:{}", new String[] {
@@ -1618,7 +1611,7 @@ public class CpRelationManager implements RelationManager {
CollectionScope collScope = new CollectionScopeImpl(
applicationScope.getApplication(),
applicationScope.getApplication(),
- CpEntityManager.getCollectionScopeNameFromEntityType( cr.getId().getType() ));
+ CpNamingUtils.getCollectionScopeNameFromEntityType( cr.getId().getType() ));
EntityCollectionManager ecm = managerCache.getEntityCollectionManager(collScope);
if ( logger.isDebugEnabled() ) {
[05/13] git commit: Fixed tests in core, Query.addEqualityFilter,
and partial update.
Posted by to...@apache.org.
Fixed tests in core, Query.addEqualityFilter, and partial update.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/d1dc7cd4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/d1dc7cd4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/d1dc7cd4
Branch: refs/heads/two-dot-o
Commit: d1dc7cd43b5f2cb9b52ee2fc0c8fc43105c25be0
Parents: 7bb9c64
Author: amuramoto <am...@apigee.com>
Authored: Wed Oct 1 15:43:05 2014 -0700
Committer: amuramoto <am...@apigee.com>
Committed: Wed Oct 1 15:43:05 2014 -0700
----------------------------------------------------------------------
.../org/apache/usergrid/corepersistence/CpEntityManager.java | 8 ++++----
.../java/org/apache/usergrid/persistence/CollectionIT.java | 3 ---
.../org/apache/usergrid/persistence/index/query/Query.java | 2 +-
.../usergrid/persistence/index/query/tree/AndOperand.java | 2 +-
.../usergrid/persistence/index/query/tree/Property.java | 2 +-
.../usergrid/persistence/index/query/tree/StringLiteral.java | 2 +-
6 files changed, 8 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d1dc7cd4/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index f62c3fd..e44f4b4 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -496,7 +496,7 @@ public class CpEntityManager implements EntityManager {
org.apache.usergrid.persistence.model.entity.Entity cpEntity =
ecm.load( entityId ).toBlockingObservable().last();
- cpEntity = CpEntityMapUtils.fromMap(
+ cpEntity = CpEntityMapUtils.fromMap(
cpEntity, entity.getProperties(), entity.getType(), true );
try {
@@ -504,8 +504,8 @@ public class CpEntityManager implements EntityManager {
cpEntity.getId().getType(), cpEntity.getId().getUuid(), cpEntity.getVersion() });
cpEntity = ecm.update( cpEntity ).toBlockingObservable().last();
- // using ecm.update() here causes Core tests to fail
-// cpEntity = ecm.write( cpEntity ).toBlockingObservable().last();
+ refreshIndex();
+ cpEntity = ecm.load( entityId ).toBlockingObservable().last();
logger.debug("Wrote {}:{} version {}", new Object[] {
cpEntity.getId().getType(), cpEntity.getId().getUuid(), cpEntity.getVersion() });
@@ -520,7 +520,7 @@ public class CpEntityManager implements EntityManager {
handleWriteUniqueVerifyException( entity, wuve );
}
}
-
+
// update in all containing collections and connection indexes
CpRelationManager rm = (CpRelationManager)getRelationManager( entity );
rm.updateContainingCollectionAndCollectionIndexes( entity, cpEntity );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d1dc7cd4/stack/core/src/test/java/org/apache/usergrid/persistence/CollectionIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/CollectionIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/CollectionIT.java
index 57b4940..a93c343 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/CollectionIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/CollectionIT.java
@@ -210,7 +210,6 @@ public class CollectionIT extends AbstractCoreIT {
assertEquals( 1, r.size() );
}
-
@Test
public void userFirstNameSearch() throws Exception {
LOG.debug( "userFirstNameSearch" );
@@ -255,9 +254,7 @@ public class CollectionIT extends AbstractCoreIT {
em.refreshIndex();
// search with the old username, should be no results
- query = new Query();
query.addEqualityFilter( "firstname", firstName );
-
r = em.searchCollection( em.getApplicationRef(), "users", query );
assertEquals( 0, r.size() );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d1dc7cd4/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java
index f845e63..a1e25da 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java
@@ -739,7 +739,7 @@ public class Query {
/** Add a equal filter to this query. && with existing clauses */
public Query addEqualityFilter( String propName, Object value ) {
- Equal equality = new Equal( new ClassicToken( 0, "=" ) );
+ Equal equality = new Equal( new ClassicToken( 1, "=" ) );
addClause( equality, propName, value );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d1dc7cd4/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/tree/AndOperand.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/tree/AndOperand.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/tree/AndOperand.java
index a6b2f8b..f0444c7 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/tree/AndOperand.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/tree/AndOperand.java
@@ -27,7 +27,7 @@ import org.apache.usergrid.persistence.index.exceptions.IndexException;
public class AndOperand extends BooleanOperand {
public AndOperand() {
- super( new CommonToken( 0, "and" ) );
+ super( new CommonToken( 1, "and" ) );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d1dc7cd4/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/tree/Property.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/tree/Property.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/tree/Property.java
index f9bf67c..e2be661 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/tree/Property.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/tree/Property.java
@@ -40,7 +40,7 @@ public class Property extends Literal<String> {
public Property( String property ) {
- this( new ClassicToken( 0, property ) );
+ this( new ClassicToken( 1, property ) );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d1dc7cd4/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/tree/StringLiteral.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/tree/StringLiteral.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/tree/StringLiteral.java
index e3ab780..92f9033 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/tree/StringLiteral.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/tree/StringLiteral.java
@@ -45,7 +45,7 @@ public class StringLiteral extends Literal<String> {
public StringLiteral( String value ) {
- super( new ClassicToken( 0, value ) );
+ super( new ClassicToken( 1, value ) );
parseValue( value );
}