You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sf...@apache.org on 2015/03/31 23:00:43 UTC
[05/50] incubator-usergrid git commit: Merge branch 'USERGRID-405'
into USERGRID-448-appinfofix
Merge branch 'USERGRID-405' into USERGRID-448-appinfofix
Conflicts:
stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
stack/core/src/test/java/org/apache/usergrid/CoreITSetupImpl.java
stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/44deb61c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/44deb61c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/44deb61c
Branch: refs/heads/USERGRID-473
Commit: 44deb61ce4789ec5b0dcee9bf8250788454424b0
Parents: e3d9cc9 c0384d5
Author: Dave Johnson <dm...@apigee.com>
Authored: Mon Mar 23 14:34:41 2015 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Mon Mar 23 14:34:41 2015 -0400
----------------------------------------------------------------------
.../batch/service/SchedulerServiceImpl.java | 17 +-
.../corepersistence/CpEntityManager.java | 41 +-
.../corepersistence/CpEntityManagerFactory.java | 70 +--
.../corepersistence/CpManagerCache.java | 6 +-
.../corepersistence/CpRelationManager.java | 19 +-
.../usergrid/corepersistence/CpSetup.java | 6 +-
.../usergrid/corepersistence/ManagerCache.java | 3 +-
.../events/EntityDeletedHandler.java | 3 +-
.../events/EntityVersionDeletedHandler.java | 4 +-
.../results/FilteringLoader.java | 3 +-
.../usergrid/persistence/EntityManager.java | 19 +-
.../cassandra/EntityManagerFactoryImpl.java | 3 +-
.../cassandra/EntityManagerImpl.java | 20 +-
.../org/apache/usergrid/CoreApplication.java | 24 +-
.../java/org/apache/usergrid/CoreITSetup.java | 6 +
.../org/apache/usergrid/CoreITSetupImpl.java | 45 +-
.../org/apache/usergrid/TestEntityIndex.java | 27 ++
.../corepersistence/StaleIndexCleanupTest.java | 87 ++--
.../usergrid/persistence/CollectionIT.java | 80 ++--
.../usergrid/persistence/CountingMutatorIT.java | 4 +-
.../persistence/EntityConnectionsIT.java | 14 +-
.../usergrid/persistence/EntityManagerIT.java | 16 +-
.../org/apache/usergrid/persistence/GeoIT.java | 26 +-
.../persistence/GeoQueryBooleanTest.java | 4 +-
.../apache/usergrid/persistence/IndexIT.java | 14 +-
.../usergrid/persistence/PathQueryIT.java | 6 +-
.../PerformanceEntityRebuildIndexTest.java | 27 +-
.../usergrid/persistence/PermissionsIT.java | 4 +-
.../cassandra/EntityManagerFactoryImplIT.java | 4 +-
.../persistence/query/CollectionIoHelper.java | 2 +-
.../persistence/query/ConnectionHelper.java | 2 +-
.../query/IntersectionTransitivePagingIT.java | 2 +-
.../query/IntersectionUnionPagingIT.java | 2 +-
.../persistence/query/IteratingQueryIT.java | 34 +-
.../resources/usergrid-custom-test.properties | 1 +
stack/corepersistence/queryindex/pom.xml | 57 +--
.../persistence/index/AliasedEntityIndex.java | 7 +-
.../index/ApplicationEntityIndex.java | 50 ++
.../usergrid/persistence/index/EntityIndex.java | 46 +-
.../persistence/index/EntityIndexBatch.java | 5 +
.../persistence/index/EntityIndexFactory.java | 2 +-
.../persistence/index/IndexIdentifier.java | 26 +-
.../usergrid/persistence/index/SearchType.java | 49 ++
.../usergrid/persistence/index/SearchTypes.java | 14 +-
.../persistence/index/guice/IndexModule.java | 5 +-
.../impl/EsApplicationEntityIndexImpl.java | 371 ++++++++++++++
.../index/impl/EsEntityIndexBatchImpl.java | 80 ++--
.../index/impl/EsEntityIndexFactoryImpl.java | 27 +-
.../index/impl/EsEntityIndexImpl.java | 478 ++++---------------
.../index/impl/EsIndexBufferConsumerImpl.java | 68 +--
.../persistence/index/impl/EsQueryVistor.java | 88 ++--
.../persistence/index/impl/IndexingUtils.java | 140 +++---
.../usergrid/persistence/index/query/Query.java | 6 +-
.../persistence/index/guice/IndexTestFig.java | 7 +-
.../index/impl/CorePerformanceIT.java | 0
.../impl/EntityConnectionIndexImplTest.java | 0
.../persistence/index/impl/EntityIndexTest.java | 181 +++----
.../index/impl/IndexLoadTestsIT.java | 328 +++++++++++--
.../usergrid/rest/AbstractContextResource.java | 102 ++--
.../org/apache/usergrid/rest/IndexResource.java | 2 +-
.../rest/test/RefreshIndexResource.java | 17 +-
.../resources/usergrid-custom-test.properties | 2 +
.../cassandra/ManagementServiceImpl.java | 1 -
.../org/apache/usergrid/ServiceApplication.java | 18 +-
.../apache/usergrid/management/EmailFlowIT.java | 6 +-
.../usergrid/management/OrganizationIT.java | 10 +-
.../org/apache/usergrid/management/RoleIT.java | 4 +-
.../cassandra/ManagementServiceIT.java | 22 +-
.../management/export/ExportServiceIT.java | 12 +-
.../management/importer/ImportCollectionIT.java | 15 +-
.../usergrid/services/ConnectionsServiceIT.java | 10 +-
.../usergrid/services/GroupServiceIT.java | 6 +-
.../usergrid/services/RolesServiceIT.java | 2 +-
.../usergrid/services/UsersServiceIT.java | 2 +-
.../AbstractServiceNotificationIT.java | 2 +-
.../apns/NotificationsServiceIT.java | 40 +-
.../gcm/NotificationsServiceIT.java | 4 +-
.../resources/usergrid-custom-test.properties | 2 +
78 files changed, 1706 insertions(+), 1253 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/44deb61c/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index 782b273,1388462..2d155a8
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@@ -15,16 -15,33 +15,33 @@@
*/
package org.apache.usergrid.corepersistence;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import com.google.inject.Injector;
+import com.google.inject.Key;
+import com.google.inject.TypeLiteral;
+
+ import java.util.Arrays;
+ import java.util.HashMap;
+ import java.util.Iterator;
+ import java.util.List;
+ import java.util.Map;
+ import java.util.TreeMap;
+ import java.util.UUID;
+ import java.util.concurrent.atomic.AtomicBoolean;
+
+ import org.apache.usergrid.persistence.index.ApplicationEntityIndex;
+ import org.slf4j.Logger;
+ import org.slf4j.LoggerFactory;
+ import org.springframework.beans.BeansException;
+ import org.springframework.context.ApplicationContext;
+ import org.springframework.context.ApplicationContextAware;
+
import org.apache.commons.lang.StringUtils;
-
import org.apache.usergrid.corepersistence.util.CpNamingUtils;
-import org.apache.usergrid.persistence.AbstractEntity;
-import org.apache.usergrid.persistence.Entity;
-import org.apache.usergrid.persistence.EntityFactory;
-import org.apache.usergrid.persistence.EntityManager;
-import org.apache.usergrid.persistence.EntityManagerFactory;
-import org.apache.usergrid.persistence.EntityRef;
-import org.apache.usergrid.persistence.Results;
+import org.apache.usergrid.exception.ConflictException;
+import org.apache.usergrid.persistence.*;
import org.apache.usergrid.persistence.cassandra.CassandraService;
import org.apache.usergrid.persistence.cassandra.CounterUtils;
import org.apache.usergrid.persistence.cassandra.Setup;
@@@ -103,9 -128,11 +121,10 @@@ public class CpEntityManagerFactory imp
this.cassandraService = cassandraService;
this.counterUtils = counterUtils;
this.injector = injector;
+ this.entityIndex = injector.getInstance(EntityIndex.class);
this.managerCache = injector.getInstance( ManagerCache.class );
this.metricsFactory = injector.getInstance( MetricsFactory.class );
-
- this.orgApplicationCache = new OrgApplicationCacheImpl( this );
+ this.applicationIdCache = new ApplicationIdCacheImpl( this );
}
@@@ -122,19 -149,21 +141,20 @@@
private void init() {
- EntityManager em = getEntityManager( CpNamingUtils.SYSTEM_APP_ID);
+ EntityManager em = getEntityManager(getManagementAppId());
try {
- if (em.getApplication() == null) {
- logger.info("Creating system application");
- Map sysAppProps = new HashMap<String, Object>();
- sysAppProps.put(PROPERTY_NAME, "systemapp");
- em.create(CpNamingUtils.SYSTEM_APP_ID, TYPE_APPLICATION, sysAppProps);
+ if ( em.getApplication() == null ) {
+ logger.info("Creating management application");
+ Map mgmtAppProps = new HashMap<String, Object>();
+ mgmtAppProps.put(PROPERTY_NAME, "systemapp");
+ em.create( getManagementAppId(), TYPE_APPLICATION, mgmtAppProps);
em.getApplication();
- em.createIndex();
- em.refreshIndex();
}
+ entityIndex.initializeIndex();
+ entityIndex.refresh();
+
-
} catch (Exception ex) {
throw new RuntimeException("Fatal error creating system application", ex);
}
@@@ -161,9 -190,9 +181,9 @@@
return entityManagers.get( applicationId );
}
catch ( Exception ex ) {
- logger.error("Error getting entity manager", ex);
+ logger.error("Error getting oldAppInfo manager", ex);
}
-- return _getEntityManager( applicationId );
++ return _getEntityManager(applicationId);
}
@@@ -230,40 -260,74 +250,44 @@@
throw new ApplicationAlreadyExistsException( appName );
}
- getSetup().setupApplicationKeyspace( applicationId, appName );
-
-
- final Optional<UUID> cachedValue = orgApplicationCache.getOrganizationId( organizationName );
-
-
- UUID orgUuid;
-
- if ( !cachedValue.isPresent() ) {
-
-
- // create new org because the specified one does not exist
- final String orgName = organizationName;
-
-
-
- try {
- final Entity orgInfo = em.create( "organization", new HashMap<String, Object>() {{
- put( PROPERTY_NAME, orgName );
- }} );
- orgUuid = orgInfo.getUuid();
- //evit so it's re-loaded later
- orgApplicationCache.evictOrgId( name );
- }
- catch ( DuplicateUniquePropertyExistsException e ) {
- //swallow, if it exists, just get it
- orgApplicationCache.evictOrgId( organizationName );
- orgUuid = orgApplicationCache.getOrganizationId( organizationName ).get();
- }
-
- } else{
- orgUuid = cachedValue.get();
- }
-
- // create appinfo entry in the system app
- final UUID appId = applicationId;
- final UUID orgId = orgUuid;
- Map<String, Object> appInfoMap = new HashMap<String, Object>() {{
- put( PROPERTY_NAME, appName );
- put( "applicationUuid", appId );
- put( "organizationUuid", orgId );
- }};
+ // create application info entity
- getSetup().setupApplicationKeyspace( applicationId, appName );
+ try {
- em.create( "appinfo", appInfoMap );
++ em.create( CpNamingUtils.APPLICATION_INFO, properties );
+ }
+ catch ( DuplicateUniquePropertyExistsException e ) {
+ throw new ApplicationAlreadyExistsException( appName );
+ }
+ entityIndex.refresh();
- // create application entity
if ( properties == null ) {
- properties = new TreeMap<String, Object>( CASE_INSENSITIVE_ORDER );
+ properties = new TreeMap<>( CASE_INSENSITIVE_ORDER );
}
properties.put( PROPERTY_NAME, appName );
- EntityManager appEm = getEntityManager( applicationId );
- appEm.create( applicationId, TYPE_APPLICATION, properties );
- appEm.createIndex();
+ EntityManager appEm = getEntityManager( applicationId);
-
+ appEm.create(applicationId, TYPE_APPLICATION, properties);
appEm.resetRoles();
- appEm.refreshIndex();
+ entityIndex.refresh();
- logger.info("Initialized application {}", appName );
+ // create application info entity in the management app
- //evict app Id from cache
- orgApplicationCache.evictAppId( appName );
+ Map<String, Object> appInfoMap = new HashMap<String, Object>() {{
+ put( PROPERTY_NAME, appName );
+ put( PROPERTY_APPLICATION_ID, applicationId );
+ }};
+
+ Entity appInfo;
+ try {
+ appInfo = em.create(CpNamingUtils.APPLICATION_INFO, appInfoMap);
+ } catch (DuplicateUniquePropertyExistsException e) {
+ throw new ApplicationAlreadyExistsException(appName);
+ }
- em.refreshIndex();
+
+ // evict app Id from cache
+ applicationIdCache.evictAppId(appName);
- return applicationId;
+ logger.info("Initialized application {}", appName);
+ return appInfo;
}
@@@ -281,88 -344,48 +305,87 @@@
@Override
public void deleteApplication(UUID applicationId) throws Exception {
- //throw new UnsupportedOperationException("Delete application not supported");
+ // find application_info for application to delete
+
+ EntityManager em = getEntityManager(getManagementAppId());
- // remove old appinfo Entity, which is in the System App's appinfos collection
- EntityManager em = getEntityManager(CpNamingUtils.SYSTEM_APP_ID);
- Query q = Query.fromQL(String.format("select * where applicationUuid = '%s'", applicationId.toString()));
- Results results = em.searchCollection(em.getApplicationRef(), "appinfos", q);
+ final Results results = em.searchCollection(em.getApplicationRef(), CpNamingUtils.APPLICATION_INFOS,
+ Query.fromQL("select * where " + PROPERTY_APPLICATION_ID + " = " + applicationId.toString()));
+ Entity appInfoToDelete = results.getEntity();
- Entity appToDelete = results.getEntity();
- if(appToDelete != null) {
- // create new Entity in deleted_appinfos collection, with same UUID and properties as deleted appinfo
- em.create("deleted_appinfo", appToDelete.getProperties());
- em.delete(appToDelete);
+ // ensure that there is not already a deleted app with the same name
+ final EntityRef alias = em.getAlias(
+ CpNamingUtils.DELETED_APPLICATION_INFO, appInfoToDelete.getName());
+ if ( alias != null ) {
+ throw new ConflictException("Cannot delete app with same name as already deleted app");
}
- entityIndex.refresh();
+
+ // make a copy of the app to delete application_info entity
+ // and put it in a deleted_application_info collection
+
+ Entity deletedApp = em.create(
+ CpNamingUtils.DELETED_APPLICATION_INFO, appInfoToDelete.getProperties());
+
+ // copy its connections too
+
+ final Set<String> connectionTypes = em.getConnectionTypes(appInfoToDelete);
+ for ( String connType : connectionTypes ) {
+ final Results connResults =
+ em.getConnectedEntities(appInfoToDelete, connType, null, Query.Level.ALL_PROPERTIES);
+ for ( Entity entity : connResults.getEntities() ) {
+ em.createConnection( deletedApp, connType, entity );
+ }
+ }
+
+ // delete the app from the application_info collection and delete its index
+
+ em.delete(appInfoToDelete);
- em.refreshIndex();
+
- final EntityIndex entityIndex = managerCache.getEntityIndex(
++ final ApplicationEntityIndex entityIndex = managerCache.getEntityIndex(
+ new ApplicationScopeImpl(new SimpleId(applicationId, TYPE_APPLICATION)));
+
+ applicationIdCache.evictAppId(appInfoToDelete.getName());
-
- entityIndex.deleteIndex();
}
@Override
- public void restoreApplication(UUID applicationId) throws Exception {
+ public Entity restoreApplication(UUID applicationId) throws Exception {
- // remove old delete_appinfos Entity
- EntityManager em = getEntityManager(CpNamingUtils.SYSTEM_APP_ID);
- Query q = Query.fromQL(String.format("select * where applicationUuid = '%s'", applicationId.toString()));
- Results results = em.searchCollection(em.getApplicationRef(), "deleted_appinfos", q);
- Entity appToRestore = results.getEntity();
+ // get the deleted_application_info for the deleted app
- if ( appToRestore == null ) {
+ EntityManager em = getEntityManager(getManagementAppId());
+
- final Results results = em.searchCollection(em.getApplicationRef(), CpNamingUtils.DELETED_APPLICATION_INFOS,
++ final Results results = em.searchCollection(
++ em.getApplicationRef(), CpNamingUtils.DELETED_APPLICATION_INFOS,
+ Query.fromQL("select * where " + PROPERTY_APPLICATION_ID + " = " + applicationId.toString()));
+ Entity deletedAppInfo = results.getEntity();
+
+ if ( deletedAppInfo == null ) {
throw new EntityNotFoundException("Cannot restore. Deleted Application not found: " + applicationId );
}
- em.delete( appToRestore );
+ // create application_info for restored app
- // restore entity in appinfo collection
- Map<String, Object> appProps = appToRestore.getProperties();
- appProps.remove("uuid");
- appProps.put("type", "appinfo");
- Entity restoredApp = em.create("appinfo", appToRestore.getProperties());
+ Entity restoredAppInfo = em.create(
+ deletedAppInfo.getUuid(), CpNamingUtils.APPLICATION_INFO, deletedAppInfo.getProperties());
+ // copy connections from deleted app entity
+
+ final Set<String> connectionTypes = em.getConnectionTypes(deletedAppInfo);
+ for ( String connType : connectionTypes ) {
+ final Results connResults =
+ em.getConnectedEntities(deletedAppInfo, connType, null, Query.Level.ALL_PROPERTIES);
+ for ( Entity entity : connResults.getEntities() ) {
+ em.createConnection( restoredAppInfo, connType, entity );
+ }
+ }
+
+ // delete the deleted app entity rebuild the app index
+
+ em.delete(deletedAppInfo);
+ entityIndex.refresh();
- // rebuild the apps index
this.rebuildApplicationIndexes(applicationId, new ProgressObserver() {
@Override
public void onProgress(EntityRef entity) {
@@@ -633,14 -664,13 +656,11 @@@
// refresh special indexes without calling EntityManager refresh because stack overflow
maybeCreateIndexes();
- // system app
+
- for ( EntityIndex index : getManagementIndexes() ) {
- index.refresh();
- }
+ entityIndex.refresh();
}
--
private void maybeCreateIndexes() {
- // system app
if ( indexInitialized.getAndSet( true ) ) {
return;
}
@@@ -651,11 -679,18 +669,11 @@@
}
- private List<EntityIndex> getManagementIndexes() {
+ private List<ApplicationEntityIndex> getManagementIndexes() {
- return Arrays.asList( managerCache.getEntityIndex(
- new ApplicationScopeImpl( new SimpleId( CpNamingUtils.SYSTEM_APP_ID, "application" ) ) ),
-
- // management app
- managerCache
- .getEntityIndex( new ApplicationScopeImpl( new SimpleId( getManagementAppId(), "application" ) ) ),
-
- // default app TODO: do we need this in two-dot-o
- managerCache
- .getEntityIndex( new ApplicationScopeImpl( new SimpleId( getDefaultAppId(), "application" ) ) ) );
+ return Arrays.asList(
+ managerCache.getEntityIndex( // management app
+ new ApplicationScopeImpl(new SimpleId(getManagementAppId(), "application"))));
}
@@@ -723,13 -750,12 +741,12 @@@
EntityManager em = getEntityManager( appId );
//explicitly invoke create index, we don't know if it exists or not in ES during a rebuild.
- em.createIndex();
Application app = em.getApplication();
-- em.reindexCollection( po, collectionName, reverse );
++ em.reindexCollection(po, collectionName, reverse);
logger.info("\n\nRebuilt index for application {} id {} collection {}\n",
- new Object[] { app.getName(), appId, collectionName } );
+ new Object[]{app.getName(), appId, collectionName});
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/44deb61c/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpSetup.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/CpSetup.java
index c5c792b,a9e6cee..4ab7874
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpSetup.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpSetup.java
@@@ -95,12 -99,24 +99,12 @@@ public class CpSetup implements Setup
setupStaticKeyspace();
//force the EMF creation of indexes before creating the default applications
- emf.refreshIndex();
+ entityIndex.initializeIndex();
- injector.getInstance( DataMigrationManager.class ).migrate();
-
- logger.info( "Setting up default applications" );
-
- try {
- emf.initializeApplication( DEFAULT_ORGANIZATION, emf.getDefaultAppId(), DEFAULT_APPLICATION, null );
- }
- catch ( ApplicationAlreadyExistsException ex ) {
- logger.warn( "Application {}/{} already exists", DEFAULT_ORGANIZATION, DEFAULT_APPLICATION );
- }
- catch ( OrganizationAlreadyExistsException oaee ) {
- logger.warn( "Organization {} already exists", DEFAULT_ORGANIZATION );
- }
+ logger.info( "Setting up management app" );
try {
- emf.initializeApplication( DEFAULT_ORGANIZATION, emf.getManagementAppId(), MANAGEMENT_APPLICATION, null );
+ emf.initializeApplicationV2( DEFAULT_ORGANIZATION, emf.getManagementAppId(), MANAGEMENT_APPLICATION, null );
}
catch ( ApplicationAlreadyExistsException ex ) {
logger.warn( "Application {}/{} already exists", DEFAULT_ORGANIZATION, MANAGEMENT_APPLICATION );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/44deb61c/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
index cc70a3c,0e1e62a..e8aae01
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
@@@ -405,10 -403,9 +405,9 @@@ public class EntityManagerFactoryImpl i
return MANAGEMENT_APPLICATION_ID;
}
-
@Override
- public UUID getDefaultAppId() {
- return DEFAULT_APPLICATION_ID;
+ public void refreshIndex() {
- // no op
++ throw new UnsupportedOperationException("Not supported.");
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/44deb61c/stack/core/src/test/java/org/apache/usergrid/CoreITSetup.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/44deb61c/stack/core/src/test/java/org/apache/usergrid/CoreITSetupImpl.java
----------------------------------------------------------------------
diff --cc stack/core/src/test/java/org/apache/usergrid/CoreITSetupImpl.java
index efa6a47,3cc0815..26254b1
--- a/stack/core/src/test/java/org/apache/usergrid/CoreITSetupImpl.java
+++ b/stack/core/src/test/java/org/apache/usergrid/CoreITSetupImpl.java
@@@ -17,32 -17,30 +17,29 @@@
package org.apache.usergrid;
--import java.util.UUID;
--
- import org.apache.usergrid.persistence.Entity;
- import org.apache.usergrid.utils.UUIDUtils;
-import org.apache.usergrid.persistence.index.EntityIndex;
--import org.junit.runner.Description;
--import org.junit.runners.model.Statement;
--import org.slf4j.Logger;
--import org.slf4j.LoggerFactory;
--
++import com.google.inject.Injector;
import org.apache.usergrid.cassandra.SpringResource;
import org.apache.usergrid.mq.QueueManagerFactory;
++import org.apache.usergrid.persistence.Entity;
import org.apache.usergrid.persistence.EntityManagerFactory;
import org.apache.usergrid.persistence.IndexBucketLocator;
import org.apache.usergrid.persistence.cassandra.CassandraService;
--import org.apache.usergrid.persistence.core.migration.data.DataMigrationManager;
--import org.apache.usergrid.persistence.core.migration.schema.MigrationException;
import org.apache.usergrid.setup.ConcurrentProcessSingleton;
import org.apache.usergrid.utils.JsonUtils;
++import org.apache.usergrid.utils.UUIDUtils;
++import org.junit.runner.Description;
++import org.junit.runners.model.Statement;
++import org.slf4j.Logger;
++import org.slf4j.LoggerFactory;
--import com.google.inject.Injector;
++import java.util.UUID;
+
+import static org.apache.usergrid.persistence.Schema.PROPERTY_APPLICATION_ID;
- public class CoreITSetupImpl implements CoreITSetup {
+ public class CoreITSetupImpl implements CoreITSetup, TestEntityIndex {
private static final Logger LOG = LoggerFactory.getLogger( CoreITSetupImpl.class );
+ private final Injector injector;
protected EntityManagerFactory emf;
protected QueueManagerFactory qmf;
@@@ -145,4 -149,24 +144,24 @@@
LOG.info( name + ":\n" + JsonUtils.mapToFormattedJsonString( obj ) );
}
}
+
+ @Override
+ public Injector getInjector() {
+ return injector;
+ }
+
+ @Override
+ public TestEntityIndex getEntityIndex(){
+ return this;
+ }
+
+ @Override
+ public void refresh(){
- try{
++ try {
+ Thread.sleep(50);
- }catch (InterruptedException ie){
++ } catch (InterruptedException ie){
+
+ }
+ getEntityIndex().refresh();
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/44deb61c/stack/core/src/test/java/org/apache/usergrid/persistence/EntityManagerIT.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/44deb61c/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
----------------------------------------------------------------------
diff --cc stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
index 8363a24,4b284df..9336df8
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
@@@ -309,13 -311,16 +310,17 @@@ public class PerformanceEntityRebuildIn
// ----------------- delete the system and application indexes
- logger.debug("Deleting app index and system app index");
+ logger.debug("Deleting app index");
deleteIndex( em.getApplicationId() );
-//
-// // deleting sytem app index will interfere with other concurrently running tests
+
++ // ----------------- test that we can read them, should fail
++
++ // deleting sytem app index will interfere with other concurrently running tests
+ //deleteIndex( CpNamingUtils.SYSTEM_APP_ID );
-//
-//
-// // ----------------- test that we can read them, should fail
-//
+
+ // ----------------- test that we can read them, should fail
+
logger.debug("Reading data, should fail this time ");
try {
readData( em, "testTypes", entityCount, 3 );
@@@ -377,12 -383,13 +383,13 @@@
Injector injector = SpringResource.getInstance().getBean( Injector.class );
EntityIndexFactory eif = injector.getInstance( EntityIndexFactory.class );
- Id appId = new SimpleId( appUuid, "application");
+ Id appId = new SimpleId( appUuid, Schema.TYPE_APPLICATION );
ApplicationScope scope = new ApplicationScopeImpl( appId );
- EntityIndex ei = eif.createEntityIndex(scope);
- EsEntityIndexImpl eeii = (EsEntityIndexImpl)ei;
+ ApplicationEntityIndex ei = eif.createApplicationEntityIndex(scope);
+
+ ei.deleteApplication().toBlocking().lastOrDefault(null);
+ app.refreshIndex();
- eeii.deleteIndex();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/44deb61c/stack/core/src/test/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImplIT.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/44deb61c/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 5de6f72,4290f23..771a11a
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@@ -18,83 -18,53 +18,44 @@@
package org.apache.usergrid.persistence.index.impl;
- import java.io.IOException;
- import java.util.ArrayList;
- import java.util.List;
- import java.util.UUID;
- import java.util.concurrent.TimeUnit;
- import java.util.concurrent.atomic.AtomicBoolean;
-
+ import com.codahale.metrics.Meter;
+ import com.codahale.metrics.Timer;
+ import com.google.common.collect.ImmutableMap;
+ import com.google.inject.Inject;
+ import com.google.inject.Singleton;
+ import org.apache.commons.lang.StringUtils;
-import org.apache.commons.lang3.ArrayUtils;
+ import org.apache.usergrid.persistence.core.future.BetterFuture;
+ import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
+ import org.apache.usergrid.persistence.core.util.Health;
+ import org.apache.usergrid.persistence.index.*;
+ import org.apache.usergrid.persistence.index.exceptions.IndexException;
-
+ import org.apache.usergrid.persistence.model.util.UUIDGenerator;
-
import org.elasticsearch.action.ActionFuture;
-
+ import org.elasticsearch.action.ShardOperationFailedException;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
--import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksRequest;
--import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksResponse;
-import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse;
-import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
-import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse;
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder;
--import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse;
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
+import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse;
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse;
- import org.elasticsearch.action.search.SearchRequestBuilder;
- import org.elasticsearch.action.search.SearchResponse;
- import org.elasticsearch.action.search.SearchScrollRequestBuilder;
-
+ import org.elasticsearch.action.admin.indices.refresh.RefreshResponse;
import org.elasticsearch.client.AdminClient;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
- import org.elasticsearch.index.query.FilterBuilder;
-import org.elasticsearch.index.query.*;
+import org.elasticsearch.index.query.MatchAllQueryBuilder;
- import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.indices.IndexAlreadyExistsException;
import org.elasticsearch.indices.IndexMissingException;
- import org.elasticsearch.search.SearchHit;
- import org.elasticsearch.search.SearchHits;
- import org.elasticsearch.search.sort.FieldSortBuilder;
- import org.elasticsearch.search.sort.SortBuilders;
- import org.elasticsearch.search.sort.SortOrder;
-
-import org.elasticsearch.indices.InvalidAliasNameException;
-import org.elasticsearch.rest.action.admin.indices.alias.delete.AliasesMissingException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
- import org.apache.commons.lang.StringUtils;
- import org.apache.commons.lang3.ArrayUtils;
-
- import org.apache.usergrid.persistence.core.future.BetterFuture;
- import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
- import org.apache.usergrid.persistence.core.scope.ApplicationScope;
- import org.apache.usergrid.persistence.core.util.Health;
- import org.apache.usergrid.persistence.core.util.ValidationUtils;
- import org.apache.usergrid.persistence.index.AliasedEntityIndex;
- import org.apache.usergrid.persistence.index.EntityIndexBatch;
- import org.apache.usergrid.persistence.index.IndexBufferProducer;
- import org.apache.usergrid.persistence.index.IndexFig;
- import org.apache.usergrid.persistence.index.IndexIdentifier;
- import org.apache.usergrid.persistence.index.IndexOperationMessage;
- import org.apache.usergrid.persistence.index.IndexScope;
- import org.apache.usergrid.persistence.index.SearchTypes;
- import org.apache.usergrid.persistence.index.exceptions.IndexException;
- import org.apache.usergrid.persistence.index.query.CandidateResult;
- import org.apache.usergrid.persistence.index.query.CandidateResults;
- import org.apache.usergrid.persistence.index.query.Query;
- import org.apache.usergrid.persistence.map.MapManager;
- import org.apache.usergrid.persistence.map.MapManagerFactory;
- import org.apache.usergrid.persistence.map.MapScope;
- import org.apache.usergrid.persistence.map.impl.MapScopeImpl;
- import org.apache.usergrid.persistence.model.entity.Id;
- import org.apache.usergrid.persistence.model.entity.SimpleId;
- import org.apache.usergrid.persistence.model.util.UUIDGenerator;
-
- import com.codahale.metrics.Timer;
- import com.google.common.base.Preconditions;
- import com.google.common.collect.ImmutableMap;
- import com.google.inject.Inject;
- import com.google.inject.assistedinject.Assisted;
-
- import static org.apache.usergrid.persistence.index.impl.IndexingUtils.BOOLEAN_PREFIX;
- import static org.apache.usergrid.persistence.index.impl.IndexingUtils.NUMBER_PREFIX;
- import static org.apache.usergrid.persistence.index.impl.IndexingUtils.SPLITTER;
- import static org.apache.usergrid.persistence.index.impl.IndexingUtils.STRING_PREFIX;
+ import java.io.IOException;
-import java.util.*;
++import java.util.Arrays;
++import java.util.HashSet;
++import java.util.List;
++import java.util.Set;
/**
@@@ -600,72 -370,16 +361,35 @@@ public class EsEntityIndexImpl implemen
}
};
- doInRetry( retryOperation );
- }
-
-
- @Override
- public int getPendingTasks() {
-
- final PendingClusterTasksResponse tasksResponse = esProvider.getClient().admin()
- .cluster().pendingClusterTasks( new PendingClusterTasksRequest() ).actionGet();
-
- return tasksResponse.pendingTasks().size();
- }
-
-
- @Override
- public CandidateResults getEntityVersions( final IndexScope scope, final Id id ) {
-
- //since we don't have paging inputs, there's no point in executing a query for paging.
-
- final String context = IndexingUtils.createContextName(scope);
- final SearchTypes searchTypes = SearchTypes.fromTypes(id.getType());
-
- final QueryBuilder queryBuilder =
- QueryBuilders.termQuery( IndexingUtils.ENTITY_CONTEXT_FIELDNAME, context );
-
- final SearchRequestBuilder srb = esProvider.getClient().prepareSearch( alias.getReadAlias() )
- .setTypes(searchTypes.getTypeNames())
- .setScroll(cursorTimeout + "m")
- .setQuery(queryBuilder);
-
- final SearchResponse searchResponse;
- try {
- //Added For Graphite Metrics
- Timer.Context timeEntityIndex = getVersionsTimer.time();
- searchResponse = srb.execute().actionGet();
- timeEntityIndex.stop();
- }
- catch ( Throwable t ) {
- logger.error( "Unable to communicate with elasticsearch" );
- failureMonitor.fail( "Unable to execute batch", t);
- throw t;
- }
-
-
- failureMonitor.success();
-
- return parseResults(searchResponse, new Query());
+ doInRetry(retryOperation);
}
+ /**
+ * Completely delete an index.
+ */
+ public void deleteIndex() {
+ AdminClient adminClient = esProvider.getClient().admin();
+
+ DeleteIndexResponse response = adminClient.indices()
- .prepareDelete( indexIdentifier.getIndex(null) ).get();
++ .prepareDelete(indexIdentifier.getIndex(null)).get();
+
- if ( response.isAcknowledged() ) {
- logger.info( "Deleted index: read {} write {}", alias.getReadAlias(), alias.getWriteAlias());
++ if (response.isAcknowledged()) {
++ logger.info("Deleted index: read {} write {}", alias.getReadAlias(), alias.getWriteAlias());
+ //invalidate the alias
+ aliasCache.invalidate(alias);
++ } else {
++ logger.info("Failed to delete index: read {} write {}", alias.getReadAlias(), alias.getWriteAlias());
+ }
- else {
- logger.info( "Failed to delete index: read {} write {}", alias.getReadAlias(), alias.getWriteAlias());
- }
++ }
++
++
+ public String[] getUniqueIndexes() {
+ Set<String> indexSet = new HashSet<>();
+ List<String> reads = Arrays.asList(getIndexes(AliasType.Read));
+ List<String> writes = Arrays.asList(getIndexes(AliasType.Write));
+ indexSet.addAll(reads);
+ indexSet.addAll(writes);
+ return indexSet.toArray(new String[0]);
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/44deb61c/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
----------------------------------------------------------------------
diff --cc stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
index 16bb703,854c3e0..cd8a404
--- a/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
@@@ -1605,84 -1636,28 +1605,83 @@@ public class ManagementServiceImpl impl
}
if ( properties == null ) {
- properties = new HashMap<String, Object>();
+ properties = new HashMap<>();
}
+ EntityManager em = emf.getEntityManager( smf.getManagementAppId() );
+
OrganizationInfo organizationInfo = getOrganizationByUuid( organizationId );
+ Entity appInfo = emf.createApplicationV2(
+ organizationInfo.getName(), applicationName, properties);
- em.refreshIndex();
- UUID applicationId = emf.createApplication( organizationInfo.getName(), applicationName, properties );
+ writeUserToken( smf.getManagementAppId(), appInfo,
+ encryptionService.plainTextCredentials(
+ generateOAuthSecretKey( AuthPrincipalType.APPLICATION ),
+ null,
+ smf.getManagementAppId() ) );
- EntityManager em = emf.getEntityManager( smf.getManagementAppId() );
- properties.put( "name", buildAppName( applicationName, organizationInfo ) );
- properties.put( "appUuid", applicationId );
- Entity appInfo = em.create( applicationId, APPLICATION_INFO, properties );
+ UUID applicationId = addApplicationToOrganization( organizationId, appInfo );
+ UserInfo user = null;
+ try {
+ user = SubjectUtils.getUser();
+ }
+ catch ( UnavailableSecurityManagerException e ) {
+ // occurs in the rare case that this is called before the full stack is initialized
+ logger.warn("Error getting user, application created activity will not be created", e);
+ }
+ if ( ( user != null ) && user.isAdminUser() ) {
+ postOrganizationActivity( organizationId, user, "create", appInfo, "Application", applicationName,
+ "<a href=\"mailto:" + user.getEmail() + "\">" + user.getName() + " (" + user.getEmail()
+ + ")</a> created a new application named " + applicationName, null );
+ }
- writeUserToken( smf.getManagementAppId(), appInfo, encryptionService
- .plainTextCredentials( generateOAuthSecretKey( AuthPrincipalType.APPLICATION ), null,
- smf.getManagementAppId() ) );
- addApplicationToOrganization( organizationId, applicationId, appInfo );
+
+ return new ApplicationInfo( applicationId, appInfo.getName() );
+ }
+
+
+ @Override
+ public void deleteApplication(UUID applicationId) throws Exception {
+ emf.deleteApplication( applicationId );
+ }
+
+
+ @Override
+ public ApplicationInfo restoreApplication(UUID applicationId) throws Exception {
+
+ ApplicationInfo app = getDeletedApplicationInfo( applicationId );
+ if ( app == null ) {
+ throw new EntityNotFoundException("Deleted application ID " + applicationId + " not found");
+ }
+
+ if ( emf.lookupApplication( app.getName() ) != null ) {
+ throw new ConflictException("Cannot restore application, one with that name already exists.");
+ }
+
+ // restore application_info entity
+
+ EntityManager em = emf.getEntityManager( emf.getManagementAppId() );
+ Entity appInfo = emf.restoreApplication(applicationId);
+
+ // restore token
+
+ writeUserToken( smf.getManagementAppId(), appInfo,
+ encryptionService.plainTextCredentials(
+ generateOAuthSecretKey( AuthPrincipalType.APPLICATION ),
+ null,
+ smf.getManagementAppId() ) );
+
+ String orgName = appInfo.getName().split("/")[0];
+ EntityRef alias = em.getAlias( Group.ENTITY_TYPE, orgName );
+ Entity orgEntity = em.get( alias );
+
+ addApplicationToOrganization( orgEntity.getUuid(), appInfo );
+
+ // create activity
UserInfo user = null;
- // if we call this method before the full stack is initialized
- // we'll get an exception
try {
user = SubjectUtils.getUser();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/44deb61c/stack/services/src/test/java/org/apache/usergrid/management/EmailFlowIT.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/44deb61c/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/44deb61c/stack/services/src/test/java/org/apache/usergrid/management/export/ExportServiceIT.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/44deb61c/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportCollectionIT.java
----------------------------------------------------------------------