You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by to...@apache.org on 2015/02/22 19:35:25 UTC
[1/5] incubator-usergrid git commit: Fixes cache bug and null
emission issue
Repository: incubator-usergrid
Updated Branches:
refs/heads/USERGRID-273 011a5e071 -> f004f5a31
Fixes cache bug and null emission issue
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/13998493
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/13998493
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/13998493
Branch: refs/heads/USERGRID-273
Commit: 139984935e8a9dce26025f390974ba7b574dfe36
Parents: d2a0ff2
Author: Todd Nine <tn...@apigee.com>
Authored: Thu Feb 19 23:29:37 2015 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Thu Feb 19 23:29:37 2015 -0700
----------------------------------------------------------------------
.../cache/CachedEntityCollectionManager.java | 2 +-
.../collection/impl/EntityCollectionManagerImpl.java | 13 ++++++++-----
.../java/org/apache/usergrid/rest/IndexResource.java | 2 +-
3 files changed, 10 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/13998493/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/cache/CachedEntityCollectionManager.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/cache/CachedEntityCollectionManager.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/cache/CachedEntityCollectionManager.java
index 5430759..186aafa 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/cache/CachedEntityCollectionManager.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/cache/CachedEntityCollectionManager.java
@@ -100,7 +100,7 @@ public class CachedEntityCollectionManager implements EntityCollectionManager {
return Observable.just( entity );
}
- return Observable.empty();
+ return targetEntityCollectionManager.load( entityId ).doOnNext( cacheAdd );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/13998493/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
index 8c754c1..7c467c6 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
@@ -230,21 +230,24 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
Preconditions.checkNotNull( entityId.getUuid(), "Entity id uuid required in load stage" );
Preconditions.checkNotNull( entityId.getType(), "Entity id type required in load stage" );
- return load( Collections.singleton( entityId ) ).map( new Func1<EntitySet, Entity>() {
+ return load( Collections.singleton( entityId ) ).flatMap( new Func1<EntitySet, Observable<Entity>>() {
@Override
- public Entity call( final EntitySet entitySet ) {
+ public Observable<Entity> call( final EntitySet entitySet ) {
final MvccEntity entity = entitySet.getEntity( entityId );
- if ( entity == null ) {
- return null;
+ if ( entity == null || !entity.getEntity().isPresent() ) {
+ return Observable.empty();
}
- return entity.getEntity().orNull();
+ return Observable.from( entity.getEntity().get() );
}
} );
}
+
+
+
@Override
public Observable<EntitySet> load( final Collection<Id> entityIds ) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/13998493/stack/rest/src/main/java/org/apache/usergrid/rest/IndexResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/IndexResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/IndexResource.java
index acce2d8..7b391a1 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/IndexResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/IndexResource.java
@@ -142,7 +142,7 @@ public class IndexResource extends AbstractContextResource {
emf.rebuildApplicationIndexes( appId, po );
}
catch ( Exception e ) {
- logger.error( "Unable to re-index application" );
+ logger.error( "Unable to re-index application", e );
}
}
};
[5/5] incubator-usergrid git commit: 1) PUT on //
will not restore a deleted app and 2) /applications?deleted=true will return
list of deleted applications.
Posted by to...@apache.org.
1) PUT on /<org-name>/<app-id> will not restore a deleted app and 2) /applications?deleted=true will return list of deleted applications.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/f004f5a3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/f004f5a3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/f004f5a3
Branch: refs/heads/USERGRID-273
Commit: f004f5a31c32bc329e859fd203315cea41a55fe9
Parents: fe8f404
Author: Dave Johnson <dm...@apigee.com>
Authored: Sat Feb 21 09:02:01 2015 -0500
Committer: Dave Johnson <dm...@apigee.com>
Committed: Sat Feb 21 09:02:01 2015 -0500
----------------------------------------------------------------------
.../corepersistence/CpEntityManagerFactory.java | 112 +++++++++++++++----
.../corepersistence/util/CpNamingUtils.java | 3 +
.../persistence/EntityManagerFactory.java | 7 ++
.../cassandra/EntityManagerFactoryImpl.java | 21 +++-
.../PerformanceEntityRebuildIndexTest.java | 16 ++-
.../cassandra/EntityManagerFactoryImplIT.java | 66 ++++++++++-
stack/pom.xml | 6 +-
stack/rest/pom.xml | 43 ++++---
.../org/apache/usergrid/rest/RootResource.java | 25 +++--
.../rest/applications/ApplicationResource.java | 72 ++++++++----
.../applications/ApplicationsResource.java | 28 +++--
11 files changed, 293 insertions(+), 106 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f004f5a3/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index 01691d4..59f2c9e 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -22,15 +22,7 @@ import com.google.inject.Injector;
import com.yammer.metrics.annotation.Metered;
import static java.lang.String.CASE_INSENSITIVE_ORDER;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeMap;
-import java.util.UUID;
+import java.util.*;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.commons.lang.StringUtils;
@@ -59,6 +51,7 @@ import org.apache.usergrid.persistence.core.util.Health;
import org.apache.usergrid.persistence.entities.Application;
import org.apache.usergrid.persistence.exceptions.ApplicationAlreadyExistsException;
import org.apache.usergrid.persistence.exceptions.DuplicateUniquePropertyExistsException;
+import org.apache.usergrid.persistence.exceptions.EntityNotFoundException;
import org.apache.usergrid.persistence.exceptions.OrganizationAlreadyExistsException;
import org.apache.usergrid.persistence.graph.Edge;
import org.apache.usergrid.persistence.graph.GraphManager;
@@ -317,6 +310,8 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
@Override
public void deleteApplication(UUID applicationId) throws Exception {
+ //throw new UnsupportedOperationException("Delete application not supported");
+
// remove old appinfo Entity, which is in the System App's appinfos collection
EntityManager em = getEntityManager(CpNamingUtils.SYSTEM_APP_ID);
Query q = Query.fromQL(String.format("select * where applicationUuid = '%s'", applicationId.toString()));
@@ -337,6 +332,43 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
@Override
+ public void restoreApplication(UUID applicationId) throws Exception {
+
+ // remove old delete_appinfos Entity
+ EntityManager em = getEntityManager(CpNamingUtils.SYSTEM_APP_ID);
+ Query q = Query.fromQL(String.format("select * where applicationUuid = '%s'", applicationId.toString()));
+ Results results = em.searchCollection( em.getApplicationRef(), "deleted_appinfos", q);
+ Entity appToRestore = results.getEntity();
+
+ if ( appToRestore == null ) {
+ throw new EntityNotFoundException("Cannot restore. Deleted Application not found: " + applicationId );
+ }
+
+ em.delete( appToRestore );
+
+ // restore entity in appinfo collection
+ Map<String, Object> appProps = appToRestore.getProperties();
+ appProps.remove("uuid");
+ appProps.put("type", "appinfo");
+ Entity restoredApp = em.create("appinfo", appToRestore.getProperties());
+
+ em.refreshIndex();
+
+ // rebuild the apps index
+ this.rebuildApplicationIndexes(applicationId, new ProgressObserver() {
+ @Override
+ public void onProgress(EntityRef entity) {
+ logger.debug("Restored entity {}:{}", entity.getType(), entity.getUuid());
+ }
+ @Override
+ public long getWriteDelayTime() {
+ return 0;
+ }
+ });
+ }
+
+
+ @Override
public UUID importApplication(
String organization, UUID applicationId,
String name, Map<String, Object> properties) throws Exception {
@@ -380,31 +412,56 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
public UUID lookupApplication( String name ) throws Exception {
init();
- EntityManager em = getEntityManager( CpNamingUtils.SYSTEM_APP_ID );
+ // TODO: why does this not work for restored apps
+
+// EntityManager em = getEntityManager( CpNamingUtils.SYSTEM_APP_ID );
+// final EntityRef alias = em.getAlias( CpNamingUtils.APPINFOS, name );
+// if ( alias == null ) {
+// return null;
+// }
+// final Entity entity = em.get( alias );
+// if ( entity == null ) {
+// return null;
+// }
+// final UUID property = ( UUID ) entity.getProperty( "applicationUuid" );
+// return property;
+ Query q = Query.fromQL( PROPERTY_NAME + " = '" + name + "'");
- final EntityRef alias = em.getAlias( CpNamingUtils.APPINFOS, name );
+ EntityManager em = getEntityManager(CpNamingUtils.SYSTEM_APP_ID);
- if ( alias == null ) {
+ Results results = em.searchCollection( em.getApplicationRef(), "appinfos", q);
+
+ if ( results.isEmpty() ) {
return null;
}
- final Entity entity = em.get( alias );
-
- if ( entity == null ) {
- return null;
+ Entity entity = results.iterator().next();
+ Object uuidObject = entity.getProperty("applicationUuid");
+ if ( uuidObject instanceof UUID ) {
+ return (UUID)uuidObject;
}
+ return UUIDUtils.tryExtractUUID( entity.getProperty("applicationUuid").toString() );
+ }
- final UUID property = ( UUID ) entity.getProperty( "applicationUuid" );
+ @Override
+ @Metered(group = "core", name = "EntityManagerFactory_getApplication")
+ public Map<String, UUID> getApplications() throws Exception {
+ return getApplications( false );
+ }
- return property;
+
+ @Override
+ @Metered(group = "core", name = "EntityManagerFactory_getApplication")
+ public Map<String, UUID> getDeletedApplications() throws Exception {
+ return getApplications( true );
}
@Override
@Metered(group = "core", name = "EntityManagerFactory_getApplication")
- public Map<String, UUID> getApplications() throws Exception {
+ public Map<String, UUID> getApplications(boolean deleted) throws Exception {
Map<String, UUID> appMap = new HashMap<String, UUID>();
@@ -415,7 +472,15 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
Application app = em.getApplication();
Id fromEntityId = new SimpleId( app.getUuid(), app.getType() );
- String edgeType = CpNamingUtils.getEdgeTypeFromCollectionName( CpNamingUtils.APPINFOS );
+ final String scopeName;
+ final String edgeType;
+ if ( deleted ) {
+ edgeType = CpNamingUtils.getEdgeTypeFromCollectionName(CpNamingUtils.DELETED_APPINFOS);
+ scopeName = CpNamingUtils.getCollectionScopeNameFromCollectionName(CpNamingUtils.DELETED_APPINFOS);
+ } else {
+ edgeType = CpNamingUtils.getEdgeTypeFromCollectionName(CpNamingUtils.APPINFOS);
+ scopeName = CpNamingUtils.getCollectionScopeNameFromCollectionName(CpNamingUtils.APPINFOS);
+ }
logger.debug("getApplications(): Loading edges of edgeType {} from {}:{}",
new Object[] { edgeType, fromEntityId.getType(), fromEntityId.getUuid() } );
@@ -436,9 +501,9 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
});
CollectionScope collScope = new CollectionScopeImpl(
- appScope.getApplication(),
- appScope.getApplication(),
- CpNamingUtils.getCollectionScopeNameFromCollectionName( CpNamingUtils.APPINFOS ));
+ appScope.getApplication(),
+ appScope.getApplication(),
+ scopeName);
org.apache.usergrid.persistence.model.entity.Entity e =
managerCache.getEntityCollectionManager( collScope ).load( targetId )
@@ -779,4 +844,5 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
return ecm.getHealth();
}
+
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f004f5a3/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
index 0d7b6ff..8208855 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
@@ -66,6 +66,9 @@ public class CpNamingUtils {
* The app infos entity object type. This holds the app name, appId, and org name
*/
public static final String APPINFOS = "appinfos";
+
+ public static final String DELETED_APPINFOS = "deleted_appinfos";
+
/**
* The name of the map that holds our entity id->type mapping
*/
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f004f5a3/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
index 6a9095d..2881111 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
@@ -19,6 +19,8 @@ package org.apache.usergrid.persistence;
import java.util.Map;
import java.util.UUID;
+
+import com.yammer.metrics.annotation.Metered;
import org.apache.usergrid.persistence.core.util.Health;
import org.apache.usergrid.persistence.index.EntityIndex;
import org.springframework.context.ApplicationContext;
@@ -95,8 +97,11 @@ public interface EntityManagerFactory {
*
* @throws Exception the exception
*/
+ @Metered(group = "core", name = "EntityManagerFactory_getApplication")
public abstract Map<String, UUID> getApplications() throws Exception;
+ public Map<String, UUID> getDeletedApplications() throws Exception;
+
public abstract void setup() throws Exception;
public abstract Map<String, String> getServiceProperties();
@@ -165,6 +170,8 @@ public interface EntityManagerFactory {
public Health getEntityStoreHealth();
+ void restoreApplication(UUID applicationId) throws Exception;
+
public interface ProgressObserver {
public void onProgress( EntityRef entity);
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f004f5a3/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
index 535a14e..05cbf5e 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
@@ -330,8 +330,7 @@ public class EntityManagerFactoryImpl implements EntityManagerFactory, Applicati
return applications;
}
-
- @Override
+ @Override
public boolean setServiceProperty( String name, String value ) {
try {
cass.setColumn( cass.getSystemKeyspace(), PROPERTIES_CF, PROPERTIES_CF, name, value );
@@ -473,6 +472,22 @@ public class EntityManagerFactoryImpl implements EntityManagerFactory, Applicati
@Override
public Health getEntityStoreHealth() {
- throw new UnsupportedOperationException("Not supported yet.");
+ throw new UnsupportedOperationException("Not supported in v1.");
+ }
+
+ @Override
+ public void restoreApplication(UUID applicationId) throws Exception {
+ throw new UnsupportedOperationException("Not supported in v1");
}
+
+ @Override
+ public Map<String, UUID> getDeletedApplications() throws Exception {
+ throw new UnsupportedOperationException("Not supported in v1");
+ }
+
+ @Override
+ public Map<String, UUID> getApplications(boolean deleted) throws Exception {
+ throw new UnsupportedOperationException("Not supported in v1");
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f004f5a3/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java b/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
index 52b6fe4..4bbce9c 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
@@ -218,8 +218,12 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT {
try {
+ // do it forwards
setup.getEmf().rebuildCollectionIndex( em.getApplicationId(), "catherders", false, po );
+ // and backwards, just to make sure both cases are covered
+ setup.getEmf().rebuildCollectionIndex( em.getApplicationId(), "catherders", true, po );
+
reporter.report();
registry.remove( meterName );
logger.info("Rebuilt index");
@@ -312,9 +316,13 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT {
// ----------------- delete the system and application indexes
logger.debug("Deleting app index and system app index");
- //deleteIndex( CpNamingUtils.SYSTEM_APP_ID );
+
deleteIndex( em.getApplicationId() );
+ // deleting sytem app index will interfere with other concurrently running tests
+ //deleteIndex( CpNamingUtils.SYSTEM_APP_ID );
+
+
// ----------------- test that we can read them, should fail
logger.debug("Reading data, should fail this time ");
@@ -335,7 +343,7 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT {
int counter = 0;
@Override
- public void onProgress( final EntityRef entity ) {
+ public void onProgress( final EntityRef entity ) {
meter.mark();
logger.debug("Indexing {}:{}", entity.getType(), entity.getUuid());
@@ -345,8 +353,6 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT {
counter++;
}
-
-
@Override
public long getWriteDelayTime() {
return 0;
@@ -355,6 +361,8 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT {
try {
+ setup.getEmf().rebuildInternalIndexes( po );
+
setup.getEmf().rebuildApplicationIndexes( em.getApplicationId(), po );
reporter.report();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f004f5a3/stack/core/src/test/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImplIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImplIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImplIT.java
index 09e7ce1..850ac6b 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImplIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImplIT.java
@@ -23,6 +23,7 @@ import java.util.List;
import java.util.Map;
import java.util.UUID;
+import org.apache.usergrid.persistence.*;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
@@ -33,11 +34,6 @@ import org.slf4j.LoggerFactory;
import org.apache.commons.lang3.RandomStringUtils;
import org.apache.usergrid.AbstractCoreIT;
-import org.apache.usergrid.persistence.Entity;
-import org.apache.usergrid.persistence.EntityManager;
-import org.apache.usergrid.persistence.EntityManagerFactory;
-import org.apache.usergrid.persistence.Results;
-import org.apache.usergrid.persistence.SimpleEntityRef;
import org.apache.usergrid.persistence.cassandra.util.TraceTag;
import org.apache.usergrid.persistence.cassandra.util.TraceTagManager;
import org.apache.usergrid.persistence.cassandra.util.TraceTagReporter;
@@ -123,10 +119,27 @@ public class EntityManagerFactoryImplIT extends AbstractCoreIT {
em.refreshIndex();
+ // TODO: this assertion should work!
+ //assertNotNull( "cannot lookup app by name", setup.getEmf().lookupApplication("test-app-" + rand) );
+
// delete the application
setup.getEmf().deleteApplication( applicationId );
+ em.refreshIndex();
+
+ boolean found = false;
+ Map<String, UUID> deletedApps = emf.getDeletedApplications();
+ for ( String appName : deletedApps.keySet() ) {
+ UUID appId = deletedApps.get( appName );
+ if ( appId.equals( applicationId )) {
+ found = true;
+ break;
+ }
+ }
+
+ assertTrue("Deleted app not found in deleted apps collection", found );
+
// attempt to get entities in application's collections in various ways should all fail
assertNull( setup.getEmf().lookupApplication("test-app-" + rand) );
@@ -138,6 +151,49 @@ public class EntityManagerFactoryImplIT extends AbstractCoreIT {
assertNotEquals( appName, "test-app-" + rand );
}
+ // restore the app
+
+ emf.restoreApplication( applicationId );
+
+ emf.rebuildAllIndexes(new EntityManagerFactory.ProgressObserver() {
+ @Override
+ public void onProgress(EntityRef entity) {
+ logger.debug("Reindexing {}:{}", entity.getType(), entity.getUuid() );
+ }
+
+ @Override
+ public long getWriteDelayTime() {
+ return 0;
+ }
+ });
+
+ // test to see that app now works and is happy
+
+ // it should not be found in the deleted apps collection
+ found = false;
+ deletedApps = emf.getDeletedApplications();
+ for ( String appName : deletedApps.keySet() ) {
+ UUID appId = deletedApps.get( appName );
+ if ( appId.equals( applicationId )) {
+ found = true;
+ break;
+ }
+ }
+ assertFalse("Restored app found in deleted apps collection", found);
+
+ found = false;
+ appMap = setup.getEmf().getApplications();
+ for ( String appName : appMap.keySet() ) {
+ UUID appId = appMap.get( appName );
+ if ( appId.equals( applicationId )) {
+ found = true;
+ break;
+ }
+ }
+ assertTrue("Restored app not found in apps collection", found);
+
+ // TODO: this assertion should work!
+ //assertNotNull(setup.getEmf().lookupApplication("test-app-" + rand));
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f004f5a3/stack/pom.xml
----------------------------------------------------------------------
diff --git a/stack/pom.xml b/stack/pom.xml
index 12fac3f..2735366 100644
--- a/stack/pom.xml
+++ b/stack/pom.xml
@@ -1565,12 +1565,14 @@
<exclude>**/aws.properties</exclude>
<exclude>**/tempExport*</exclude>
<exclude>loadtests/loadtest_setup.sh</exclude>
- <exclude>loadtests/gatling/user-files/request-bodies/**</exclude>
+ <exclude>loadtests/gatling/user-files/request-bodies/**</exclude>
+
<!-- other -->
<exclude>**/m2/**</exclude>
<exclude>**/*.asc</exclude>
<exclude>**/dummy.txt</exclude>
- <exclude>**/cloudbees.xml</exclude>
+ <exclude>**/cloudbees.xml</exclude>
+ <exclude>**/catalina_base/**</exclude>
<exlude>loadtests/gatling/lib/**</exlude>
<exlude>loadtests/gatling/user-files/**</exlude>
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f004f5a3/stack/rest/pom.xml
----------------------------------------------------------------------
diff --git a/stack/rest/pom.xml b/stack/rest/pom.xml
index e07e218..0680da2 100644
--- a/stack/rest/pom.xml
+++ b/stack/rest/pom.xml
@@ -45,12 +45,11 @@
<id>arquillian-tomcat</id>
<activation>
- <activeByDefault>true</activeByDefault>
+ <activeByDefault>false</activeByDefault>
</activation>
<dependencies>
-
<!--embedded mode-->
<!--<dependency>-->
<!--<groupId>org.apache.tomcat.embed</groupId>-->
@@ -429,26 +428,26 @@
<!-- Some arquillian dependency runs and old version of guice. We're overridding it here so that we include
the right value into the test scope -->
- <dependency>
- <groupId>com.google.inject</groupId>
- <artifactId>guice</artifactId>
- <version>${guice.version}</version>
- <scope>test</scope>
- </dependency>
+ <!--<dependency>-->
+ <!--<groupId>com.google.inject</groupId>-->
+ <!--<artifactId>guice</artifactId>-->
+ <!--<version>${guice.version}</version>-->
+ <!--<scope>test</scope>-->
+ <!--</dependency>-->
- <dependency>
- <groupId>com.google.inject.extensions</groupId>
- <artifactId>guice-multibindings</artifactId>
- <version>${guice.version}</version>
- <scope>test</scope>
- </dependency>
+ <!--<dependency>-->
+ <!--<groupId>com.google.inject.extensions</groupId>-->
+ <!--<artifactId>guice-multibindings</artifactId>-->
+ <!--<version>${guice.version}</version>-->
+ <!--<scope>test</scope>-->
+ <!--</dependency>-->
- <dependency>
- <groupId>com.google.inject.extensions</groupId>
- <artifactId>guice-assistedinject</artifactId>
- <version>${guice.version}</version>
- <scope>test</scope>
- </dependency>
+ <!--<dependency>-->
+ <!--<groupId>com.google.inject.extensions</groupId>-->
+ <!--<artifactId>guice-assistedinject</artifactId>-->
+ <!--<version>${guice.version}</version>-->
+ <!--<scope>test</scope>-->
+ <!--</dependency>-->
<!-- documentation here
@@ -463,10 +462,6 @@
<type>pom</type>
</dependency>
-
-
-
-
<!-- use the external test client. Just depend on the maven jetty plugin to launch jetty -->
<dependency>
<groupId>com.sun.jersey.jersey-test-framework</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f004f5a3/stack/rest/src/main/java/org/apache/usergrid/rest/RootResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/RootResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/RootResource.java
index f324d28..27eff28 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/RootResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/RootResource.java
@@ -111,9 +111,10 @@ public class RootResource extends AbstractContextResource implements MetricProce
@RequireSystemAccess
@GET
@Path("applications")
- public JSONWithPadding getAllApplications( @Context UriInfo ui,
- @QueryParam("callback") @DefaultValue("callback") String callback )
- throws URISyntaxException {
+ public JSONWithPadding getAllApplications(
+ @Context UriInfo ui,
+ @QueryParam("deleted") @DefaultValue("false") Boolean deleted,
+ @QueryParam("callback") @DefaultValue("callback") String callback ) throws URISyntaxException {
logger.info( "RootResource.getAllApplications" );
@@ -122,7 +123,11 @@ public class RootResource extends AbstractContextResource implements MetricProce
Map<String, UUID> applications = null;
try {
- applications = emf.getApplications();
+ if ( deleted ) {
+ applications = emf.getDeletedApplications();
+ } else {
+ applications = emf.getApplications();
+ }
response.setSuccess();
response.setApplications( applications );
}
@@ -141,7 +146,7 @@ public class RootResource extends AbstractContextResource implements MetricProce
public JSONWithPadding getAllApplications2( @Context UriInfo ui,
@QueryParam("callback") @DefaultValue("callback") String callback )
throws URISyntaxException {
- return getAllApplications( ui, callback );
+ return getAllApplications( ui, false, callback );
}
@@ -162,16 +167,16 @@ public class RootResource extends AbstractContextResource implements MetricProce
/**
* Return status of this Usergrid instance in JSON format.
- *
+ *
* By Default this end-point will ignore errors but if you call it with ignore_status=false
* then it will return HTTP 500 if either the Entity store or the Index for the management
* application are in a bad state.
- *
+ *
* @param ignoreError Ignore any errors and return status no matter what.
*/
@GET
@Path("status")
- public JSONWithPadding getStatus(
+ public JSONWithPadding getStatus(
@QueryParam("ignore_error") @DefaultValue("true") Boolean ignoreError,
@QueryParam("callback") @DefaultValue("callback") String callback ) {
@@ -195,10 +200,10 @@ public class RootResource extends AbstractContextResource implements MetricProce
node.put( "version", usergridSystemMonitor.getBuildNumber() );
// Hector status, for backwards compatibility
- node.put( "cassandraAvailable", usergridSystemMonitor.getIsCassandraAlive() );
+ node.put( "cassandraAvailable", usergridSystemMonitor.getIsCassandraAlive() );
// Core Persistence Collections module status
- node.put( "cassandraStatus", emf.getEntityStoreHealth().toString() );
+ node.put( "cassandraStatus", emf.getEntityStoreHealth().toString() );
// Core Persistence Query Index module status for Management App Index
EntityManager em = emf.getEntityManager( emf.getManagementAppId() );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f004f5a3/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ApplicationResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ApplicationResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ApplicationResource.java
index 6a3c94c..2c252e7 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ApplicationResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ApplicationResource.java
@@ -20,18 +20,10 @@ package org.apache.usergrid.rest.applications;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.Map;
+import java.util.Properties;
import java.util.UUID;
-import javax.ws.rs.Consumes;
-import javax.ws.rs.DELETE;
-import javax.ws.rs.DefaultValue;
-import javax.ws.rs.FormParam;
-import javax.ws.rs.GET;
-import javax.ws.rs.HeaderParam;
-import javax.ws.rs.POST;
-import javax.ws.rs.Path;
-import javax.ws.rs.Produces;
-import javax.ws.rs.QueryParam;
+import javax.ws.rs.*;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.PathSegment;
@@ -328,9 +320,9 @@ public class ApplicationResource extends ServiceResource {
@POST
@Path("token")
@Consumes(APPLICATION_JSON)
- public Response getAccessTokenPostJson( @Context UriInfo ui,
- @HeaderParam("Authorization") String authorization,
- Map<String, Object> json,
+ public Response getAccessTokenPostJson( @Context UriInfo ui,
+ @HeaderParam("Authorization") String authorization,
+ Map<String, Object> json,
@QueryParam("callback") @DefaultValue("") String callback ) throws Exception {
String grant_type = ( String ) json.get( "grant_type" );
@@ -352,7 +344,7 @@ public class ApplicationResource extends ServiceResource {
}
}
- return getAccessToken( ui, authorization, grant_type, username, password, pin, client_id,
+ return getAccessToken( ui, authorization, grant_type, username, password, pin, client_id,
client_secret, code, ttl, redirect_uri, callback );
}
@@ -384,7 +376,7 @@ public class ApplicationResource extends ServiceResource {
@Path("credentials")
@RequireApplicationAccess
@Produces(MediaType.APPLICATION_JSON)
- public JSONWithPadding generateKeys( @Context UriInfo ui,
+ public JSONWithPadding generateKeys( @Context UriInfo ui,
@QueryParam("callback") @DefaultValue("callback") String callback ) throws Exception {
logger.debug( "AuthResource.keys" );
@@ -393,7 +385,7 @@ public class ApplicationResource extends ServiceResource {
throw new UnauthorizedException();
}
- ClientCredentialsInfo kp = new ClientCredentialsInfo(
+ ClientCredentialsInfo kp = new ClientCredentialsInfo(
management.getClientIdForApplication( services.getApplicationId() ),
management.newClientSecretForApplication( services.getApplicationId() ) );
@@ -405,12 +397,12 @@ public class ApplicationResource extends ServiceResource {
@GET
@Path("authorize")
- public Viewable showAuthorizeForm(
- @Context UriInfo ui,
+ public Viewable showAuthorizeForm(
+ @Context UriInfo ui,
@QueryParam("response_type") String response_type,
@QueryParam("client_id") String client_id,
@QueryParam("redirect_uri") String redirect_uri,
- @QueryParam("scope") String scope,
+ @QueryParam("scope") String scope,
@QueryParam("state") String state ) {
try {
@@ -443,7 +435,7 @@ public class ApplicationResource extends ServiceResource {
@POST
@Path("authorize")
@Produces(MediaType.TEXT_HTML)
- public Response handleAuthorizeForm( @Context UriInfo ui,
+ public Response handleAuthorizeForm( @Context UriInfo ui,
@FormParam("response_type") String response_type,
@FormParam("client_id") String client_id,
@FormParam("redirect_uri") String redirect_uri,
@@ -501,21 +493,55 @@ public class ApplicationResource extends ServiceResource {
}
+ @PUT
+ @RequireOrganizationAccess
+ @Override
+ public JSONWithPadding executePut( @Context UriInfo ui, String body,
+ @QueryParam("callback") @DefaultValue("callback") String callback ) throws Exception {
+
+ if ( applicationId == null ) {
+ throw new IllegalArgumentException("Application ID not specified in request");
+ }
+
+ ApplicationInfo app = management.getApplicationInfo( applicationId );
+ if ( app == null ) {
+ throw new EntityNotFoundException("Application ID " + applicationId + " not found");
+ }
+
+ emf.restoreApplication( applicationId );
+
+ ApiResponse response = createApiResponse();
+ response.setAction( "restore" );
+ response.setApplication( services.getApplication() );
+ response.setParams( ui.getQueryParameters() );
+
+ return new JSONWithPadding( response, callback );
+ }
+
+
@DELETE
@RequireOrganizationAccess
@Override
public JSONWithPadding executeDelete( @Context UriInfo ui,
@QueryParam("callback") @DefaultValue("callback") String callback ) throws Exception {
-
+
+ Properties props = management.getProperties();
+
+ // for now, only works in test mode
+ String testProp = ( String ) props.get( "usergrid.test" );
+ if ( testProp == null || !Boolean.parseBoolean( testProp ) ) {
+ throw new UnsupportedOperationException();
+ }
+
if ( applicationId == null ) {
throw new IllegalArgumentException("Application ID not specified in request");
}
-
+
ApplicationInfo app = management.getApplicationInfo( applicationId );
if ( app == null ) {
throw new EntityNotFoundException("Application ID " + applicationId + " not found");
}
-
+
emf.deleteApplication( applicationId );
ApiResponse response = createApiResponse();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f004f5a3/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/ApplicationsResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/ApplicationsResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/ApplicationsResource.java
index 1d2c504..56d2f98 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/ApplicationsResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/ApplicationsResource.java
@@ -21,15 +21,7 @@ import java.util.LinkedHashMap;
import java.util.Map;
import java.util.UUID;
-import javax.ws.rs.Consumes;
-import javax.ws.rs.DefaultValue;
-import javax.ws.rs.FormParam;
-import javax.ws.rs.GET;
-import javax.ws.rs.POST;
-import javax.ws.rs.Path;
-import javax.ws.rs.PathParam;
-import javax.ws.rs.Produces;
-import javax.ws.rs.QueryParam;
+import javax.ws.rs.*;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.UriInfo;
@@ -78,9 +70,10 @@ public class ApplicationsResource extends AbstractContextResource {
@RequireOrganizationAccess
@GET
- public JSONWithPadding getOrganizationApplications( @Context UriInfo ui,
- @QueryParam( "callback" ) @DefaultValue( "callback" )
- String callback ) throws Exception {
+ public JSONWithPadding getOrganizationApplications(
+ @Context UriInfo ui,
+ @QueryParam( "deleted" ) @DefaultValue( "false" ) Boolean deleted, // only return deleted apps if true
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback ) throws Exception {
ApiResponse response = createApiResponse();
response.setAction( "get organization application" );
@@ -148,6 +141,17 @@ public class ApplicationsResource extends AbstractContextResource {
@RequireOrganizationAccess
+ @Path(RootResource.APPLICATION_ID_PATH)
+ public ApplicationResource restoreApplicationFromOrganizationByApplicationId(
+ @Context UriInfo ui,
+ @PathParam( "applicationId" )
+ String applicationIdStr ) throws Exception {
+
+ return getSubResource( ApplicationResource.class ).init( organization, UUID.fromString( applicationIdStr ) );
+ }
+
+
+ @RequireOrganizationAccess
@Path( "{applicationName}" )
public ApplicationResource applicationFromOrganizationByApplicationName( @Context UriInfo ui,
@PathParam( "applicationName" )
[4/5] incubator-usergrid git commit: Merge branch 'USERGRID-273' into
app-rebuild-fix
Posted by to...@apache.org.
Merge branch 'USERGRID-273' into app-rebuild-fix
Conflicts:
stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
stack/services/src/test/java/org/apache/usergrid/management/importer/ImportCollectionIT.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/fe8f4044
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/fe8f4044
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/fe8f4044
Branch: refs/heads/USERGRID-273
Commit: fe8f4044a55e6d43373aaba8e0ee9be312d2a2f1
Parents: 2f077f1 011a5e0
Author: Dave Johnson <dm...@apigee.com>
Authored: Fri Feb 20 15:31:35 2015 -0500
Committer: Dave Johnson <dm...@apigee.com>
Committed: Fri Feb 20 15:31:35 2015 -0500
----------------------------------------------------------------------
.../src/main/groovy/configure_usergrid.groovy | 2 +-
.../main/resources/usergrid-default.properties | 15 +-
.../usergrid/corepersistence/CpSetup.java | 3 +
.../entities/FailedImportEntity.java | 11 +-
.../EntityCollectionManagerFactoryImpl.java | 37 ++--
.../collection/mvcc/stage/write/WriteStart.java | 31 ++--
.../mvcc/stage/TestEntityGenerator.java | 54 +++---
.../mvcc/stage/write/WriteStartTest.java | 51 +++++-
.../persistence/model/entity/Entity.java | 5 +
.../datagenerators/EntityDataGenerator.scala | 17 +-
.../datagenerators/FeederGenerator.scala | 24 ++-
.../usergrid/scenarios/EntityScenarios.scala | 20 +-
.../org/apache/usergrid/settings/Settings.scala | 2 +
.../PostCustomEntitySimulation.scala | 27 ++-
stack/mongo-emulator/pom.xml | 15 +-
stack/pom.xml | 63 ++-----
stack/rest/README.md | 14 +-
stack/rest/pom.xml | 20 +-
.../org/apache/usergrid/rest/IndexResource.java | 50 +++++
.../usergrid/rest/filters/MeteringFilter.java | 2 +
.../security/SecuredResourceFilterFactory.java | 2 +
.../security/shiro/filters/SecurityFilter.java | 3 +
.../apache/usergrid/rest/AbstractRestIT.java | 9 +-
.../usergrid/rest/DatabaseInitializer.java | 146 ---------------
.../rest/management/OrganizationsIT.java | 2 +-
.../test/resource2point0/AbstractRestIT.java | 22 +++
stack/rest/src/test/resources/arquillian.xml | 24 ++-
.../resources/usergrid-deployment.properties | 2 -
.../management/export/S3ExportImpl.java | 6 +-
.../management/importer/ImportServiceImpl.java | 5 +-
.../management/importer/S3ImportImpl.java | 28 ++-
.../services/assets/data/BinaryStore.java | 3 +-
.../services/assets/data/S3BinaryStore.java | 18 +-
.../org/apache/usergrid/ServiceITSetupImpl.java | 1 +
.../usergrid/management/OrganizationIT.java | 30 +--
.../management/export/ExportServiceIT.java | 183 ++++++++++++-------
.../importer/FileImportTrackerTest.java | 55 +++---
.../management/importer/ImportCollectionIT.java | 73 +++++---
.../usergrid/management/importer/S3Upload.java | 18 +-
.../services/ApplicationsServiceIT.java | 8 +-
.../usergrid/services/ServiceInvocationIT.java | 7 +
.../src/test/resources/project.properties | 1 +
.../usergrid/cassandra/ClearShiroSubject.java | 18 ++
43 files changed, 626 insertions(+), 501 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fe8f4044/stack/rest/src/main/java/org/apache/usergrid/rest/IndexResource.java
----------------------------------------------------------------------
diff --cc stack/rest/src/main/java/org/apache/usergrid/rest/IndexResource.java
index 251e25c,acce2d8..85f27ed
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/IndexResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/IndexResource.java
@@@ -193,9 -186,9 +193,59 @@@ public class IndexResource extends Abst
response.setSuccess();
return new JSONWithPadding( response, callback );
+ }
+
+ @RequireSystemAccess
++ @PUT
++ @Path( "rebuildinternal" )
++ public JSONWithPadding rebuildInternalIndexes(
++ @Context UriInfo ui,
++ @PathParam( "applicationId" ) String applicationIdStr,
++ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback,
++ @QueryParam( "delay" ) @DefaultValue( "10" ) final long delay ) throws Exception {
++
++
++ final UUID appId = UUIDUtils.tryExtractUUID(applicationIdStr);
++ ApiResponse response = createApiResponse();
++ response.setAction( "rebuild indexes started" );
++
++ final EntityManagerFactory.ProgressObserver po = new EntityManagerFactory.ProgressObserver() {
++
++ @Override
++ public void onProgress( final EntityRef entity ) {
++ logger.info( "Indexing entity {}:{}", entity.getType(), entity.getUuid() );
++ }
++
++
++ @Override
++ public long getWriteDelayTime() {
++ return delay;
++ }
++ };
++
++ final Thread rebuild = new Thread() {
++
++ @Override
++ public void run() {
++ try {
++ emf.rebuildInternalIndexes( po );
++ }
++ catch ( Exception e ) {
++ logger.error( "Unable to re-index internals", e );
++ }
++ }
++ };
++
++ rebuild.setName( String.format( "Index rebuild for app %s", appId ) );
++ rebuild.setDaemon( true );
++ rebuild.start();
++
++ response.setSuccess();
++
++ return new JSONWithPadding( response, callback );
+ }
+
+ @RequireSystemAccess
@POST
@Path( RootResource.APPLICATION_ID_PATH )
public JSONWithPadding addIndex(@Context UriInfo ui,
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fe8f4044/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportCollectionIT.java
----------------------------------------------------------------------
diff --cc stack/services/src/test/java/org/apache/usergrid/management/importer/ImportCollectionIT.java
index 7fcfc57,acd3077..d10b611
--- a/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportCollectionIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportCollectionIT.java
@@@ -110,8 -110,9 +110,9 @@@ public class ImportCollectionIT
bucketPrefix = System.getProperty( "bucketName" );
// start the scheduler after we're all set up
- JobSchedulerService jobScheduler = ConcurrentProcessSingleton
- .getInstance().getSpringResource().getBean( JobSchedulerService.class );
+ JobSchedulerService jobScheduler = ConcurrentProcessSingleton.getInstance()
+ .getSpringResource().getBean( JobSchedulerService.class );
+
if ( jobScheduler.state() != Service.State.RUNNING ) {
jobScheduler.startAsync();
jobScheduler.awaitRunning();
@@@ -480,14 -499,14 +499,14 @@@
logger.debug("\n\nImport into new app {}\n", em.getApplication().getName() );
-- ImportService importService = setup.getImportService();
++ final ImportService importService = setup.getImportService();
-- Import importEntity = importService.schedule(em.getApplication().getUuid(),
++ final Import importEntity = importService.schedule(em.getApplication().getUuid(),
new HashMap<String, Object>() {{
- put( "path", organization.getName() + em.getApplication().getName() );
- put( "organizationId", organization.getUuid() );
- put( "applicationId", em.getApplication().getUuid() );
- put( "properties", new HashMap<String, Object>() {{
+ put( "path", organization.getName() + em.getApplication().getName() );
+ put( "organizationId", organization.getUuid() );
+ put( "applicationId", em.getApplication().getUuid() );
+ put( "properties", new HashMap<String, Object>() {{
put( "storage_provider", "s3" );
put( "storage_info", new HashMap<String, Object>() {{
put( "s3_access_id",
[3/5] incubator-usergrid git commit: reindexCollection now
implemented w/test in the Core module. Still needs a REST level test.
Posted by to...@apache.org.
reindexCollection now implemented w/test in the Core module. Still needs a REST level test.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/2f077f1b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/2f077f1b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/2f077f1b
Branch: refs/heads/USERGRID-273
Commit: 2f077f1bada2cb79db5dc9af19f101b668f92b9c
Parents: a704ed7
Author: Dave Johnson <dm...@apigee.com>
Authored: Fri Feb 20 13:52:21 2015 -0500
Committer: Dave Johnson <dm...@apigee.com>
Committed: Fri Feb 20 13:52:21 2015 -0500
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 37 ++++----
.../usergrid/corepersistence/CpWalker.java | 20 +++-
.../usergrid/persistence/EntityManager.java | 2 +-
.../cassandra/EntityManagerFactoryImpl.java | 11 +--
.../cassandra/EntityManagerImpl.java | 4 +-
.../PerformanceEntityRebuildIndexTest.java | 97 ++++++++++++--------
.../management/importer/ImportCollectionIT.java | 6 +-
7 files changed, 105 insertions(+), 72 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2f077f1b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 6af0539..721ac80 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -2745,11 +2745,12 @@ public class CpEntityManager implements EntityManager {
*/
@Override
public void reindexCollection(
- final EntityManagerFactory.ProgressObserver po, String collectionName) throws Exception {
+ final EntityManagerFactory.ProgressObserver po, String collectionName, boolean reverse) throws Exception {
CpWalker walker = new CpWalker( );
- walker.walkCollections( this, application, collectionName, new CpVisitor() {
+ walker.walkCollections(
+ this, application, collectionName, reverse, new CpVisitor() {
@Override
public void visitCollectionEntry( EntityManager em, String collName, Entity entity ) {
@@ -2759,9 +2760,9 @@ public class CpEntityManager implements EntityManager {
po.onProgress( entity );
}
catch ( WriteOptimisticVerifyException wo ) {
- //swallow this, it just means this was already updated, which accomplishes our task. Just ignore.
- logger.warn( "Someone beat us to updating entity {} in collection {}. Ignoring.", entity.getName(),
- collName );
+ // swallow this, it just means this was already updated, which accomplishes our task
+ logger.warn( "Someone beat us to updating entity {} in collection {}. Ignoring.",
+ entity.getName(), collName );
}
catch ( Exception ex ) {
logger.error( "Error repersisting entity", ex );
@@ -2778,23 +2779,23 @@ public class CpEntityManager implements EntityManager {
CpWalker walker = new CpWalker( );
- walker.walkCollections( this, application, null, new CpVisitor() {
+ walker.walkCollections( this, application, null, false, new CpVisitor() {
@Override
public void visitCollectionEntry( EntityManager em, String collName, Entity entity ) {
- try {
- em.update( entity );
- po.onProgress( entity );
- }
- catch ( WriteOptimisticVerifyException wo ) {
- //swallow this, it just means this was already updated, which accomplishes our task. Just ignore.
- logger.warn( "Someone beat us to updating entity {} in collection {}. Ignoring.", entity.getName(),
- collName );
- }
- catch ( Exception ex ) {
- logger.error( "Error repersisting entity", ex );
- }
+ try {
+ em.update( entity );
+ po.onProgress( entity );
+ }
+ catch ( WriteOptimisticVerifyException wo ) {
+ //swallow this, it just means this was already updated, which accomplishes our task.
+ logger.warn( "Someone beat us to updating entity {} in collection {}. Ignoring.",
+ entity.getName(), collName );
+ }
+ catch ( Exception ex ) {
+ logger.error( "Error repersisting entity", ex );
+ }
}
} );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2f077f1b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpWalker.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpWalker.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpWalker.java
index 6bd90ec..fab7e16 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpWalker.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpWalker.java
@@ -56,15 +56,17 @@ public class CpWalker {
public void walkCollections(final CpEntityManager em, final EntityRef start,
- String collectionName, final CpVisitor visitor) throws Exception {
+ String collectionName, boolean reverse, final CpVisitor visitor) throws Exception {
- doWalkCollections( em, collectionName, new SimpleId( start.getUuid(), start.getType() ), visitor );
+ doWalkCollections(
+ em, collectionName, reverse, new SimpleId( start.getUuid(), start.getType() ), visitor );
}
private void doWalkCollections(
final CpEntityManager em,
final String collectionName,
+ final boolean reverse,
final Id applicationId,
final CpVisitor visitor ) {
@@ -80,12 +82,20 @@ public class CpWalker {
applicationScope.getApplication().getUuid()
} );
- // only search edge types that start with collections
+ final SearchByEdgeType.Order order;
+ if ( reverse ) {
+ order = SearchByEdgeType.Order.ASCENDING;
+ } else {
+ order = SearchByEdgeType.Order.DESCENDING;
+ }
final String edgeType;
- if ( collectionName != null ) {
+ if ( collectionName == null ) {
+ // only search edge types that end with collections suffix
edgeType = CpNamingUtils.EDGE_COLL_SUFFIX;
+
} else {
+ // only search edges to one collection
edgeType = CpNamingUtils.getEdgeTypeFromCollectionName( collectionName );
}
@@ -99,7 +109,7 @@ public class CpWalker {
logger.debug( "Loading edges of type {} from node {}", edgeType, applicationId );
return gm.loadEdgesFromSource( new SimpleSearchByEdgeType(
- applicationId, edgeType, Long.MAX_VALUE, SearchByEdgeType.Order.DESCENDING, null ) );
+ applicationId, edgeType, Long.MAX_VALUE, order , null ) );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2f077f1b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
index b7ca477..4054d77 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
@@ -704,7 +704,7 @@ public interface EntityManager {
public void flushManagerCaches();
void reindexCollection(
- EntityManagerFactory.ProgressObserver po, String collectionName) throws Exception;
+ EntityManagerFactory.ProgressObserver po, String collectionName, boolean reverse) throws Exception;
public void reindex( final EntityManagerFactory.ProgressObserver po ) throws Exception;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2f077f1b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
index b7808d8..535a14e 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
@@ -423,6 +423,11 @@ public class EntityManagerFactoryImpl implements EntityManagerFactory, Applicati
}
@Override
+ public void rebuildCollectionIndex(UUID appId, String collection, boolean reverse, ProgressObserver po) throws Exception {
+ throw new UnsupportedOperationException("Not supported.");
+ }
+
+ @Override
public void rebuildInternalIndexes(ProgressObserver po) throws Exception {
throw new UnsupportedOperationException("Not supported.");
}
@@ -461,12 +466,6 @@ public class EntityManagerFactoryImpl implements EntityManagerFactory, Applicati
throw new UnsupportedOperationException("Not supported in v1");
}
-
- @Override
- public void rebuildCollectionIndex(UUID appId, String collection, ProgressObserver po) {
- throw new UnsupportedOperationException("Not supported.");
- }
-
@Override
public void addIndex(UUID appId, String suffix,final int shards,final int replicas) {
throw new UnsupportedOperationException("Not supported in v1");
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2f077f1b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
index d0fe985..be43920 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
@@ -2932,7 +2932,9 @@ public class EntityManagerImpl implements EntityManager {
}
@Override
- public void reindexCollection(EntityManagerFactory.ProgressObserver po, String collectionName) throws Exception {
+ public void reindexCollection(
+ EntityManagerFactory.ProgressObserver po, String collectionName, boolean reverse) throws Exception {
+
throw new UnsupportedOperationException("Not supported.");
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2f077f1b/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java b/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
index e853e94..52b6fe4 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
@@ -24,6 +24,7 @@ import java.util.Map;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
+import org.apache.commons.lang.RandomStringUtils;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -60,7 +61,7 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT {
private static final MetricRegistry registry = new MetricRegistry();
private Slf4jReporter reporter;
- private static final long RUNTIME_MS = TimeUnit.SECONDS.toMillis( 3 );
+ private static final long RUNTIME_MS = TimeUnit.SECONDS.toMillis( 10 );
private static final long WRITE_DELAY_MS = 10;
@@ -92,7 +93,10 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT {
logger.info("Started rebuildIndex()");
- final EntityManager em = app.getEntityManager();
+ String rand = RandomStringUtils.randomAlphanumeric(5);
+ final UUID appId = setup.createApplication("org_" + rand, "app_" + rand);
+
+ final EntityManager em = setup.getEmf().getEntityManager( appId );
// ----------------- create a bunch of entities
@@ -101,40 +105,50 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT {
put("key2", 2000 );
put("key3", "Some value");
}};
- Map<String, Object> cat1map = new HashMap<String, Object>() {{
- put("name", "enzo");
- put("color", "orange");
- }};
- Map<String, Object> cat2map = new HashMap<String, Object>() {{
- put("name", "marquee");
- put("color", "grey");
- }};
- Map<String, Object> cat3map = new HashMap<String, Object>() {{
- put("name", "bertha");
- put("color", "tabby");
- }};
- Entity cat1 = em.create("cat", cat1map );
- Entity cat2 = em.create("cat", cat2map );
- Entity cat3 = em.create("cat", cat3map );
+// Map<String, Object> cat1map = new HashMap<String, Object>() {{
+// put("name", "enzo");
+// put("color", "orange");
+// }};
+// Map<String, Object> cat2map = new HashMap<String, Object>() {{
+// put("name", "marquee");
+// put("color", "grey");
+// }};
+// Map<String, Object> cat3map = new HashMap<String, Object>() {{
+// put("name", "bertha");
+// put("color", "tabby");
+// }};
+//
+// Entity cat1 = em.create("cat", cat1map );
+// Entity cat2 = em.create("cat", cat2map );
+// Entity cat3 = em.create("cat", cat3map );
final long stopTime = System.currentTimeMillis() + RUNTIME_MS;
List<EntityRef> entityRefs = new ArrayList<EntityRef>();
int entityCount = 0;
+ int herderCount = 0;
+ int shepardCount = 0;
while ( System.currentTimeMillis() < stopTime ) {
final Entity entity;
try {
entityMap.put("key", entityCount );
- entity = em.create("testType", entityMap );
+
+ if ( entityCount % 2 == 0 ) {
+ entity = em.create("catherder", entityMap);
+ herderCount++;
+ } else {
+ entity = em.create("catshepard", entityMap);
+ shepardCount++;
+ }
em.refreshIndex();
- em.createConnection(entity, "herds", cat1);
- em.createConnection(entity, "herds", cat2);
- em.createConnection(entity, "herds", cat3);
+// em.createConnection(entity, "herds", cat1);
+// em.createConnection(entity, "herds", cat2);
+// em.createConnection(entity, "herds", cat3);
} catch (Exception ex) {
throw new RuntimeException("Error creating entity", ex);
@@ -155,24 +169,25 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT {
// ----------------- test that we can read them, should work fine
logger.debug("Read the data");
- readData("testTypes", entityCount );
+ readData( em, "catherders", herderCount, 0);
+ readData( em, "catshepards", shepardCount, 0);
// ----------------- delete the system and application indexes
- logger.debug("Deleting app index and system app index");
- deleteIndex( CpNamingUtils.SYSTEM_APP_ID );
+ logger.debug("Deleting app index index");
+ //deleteIndex( CpNamingUtils.SYSTEM_APP_ID );
deleteIndex( em.getApplicationId() );
// ----------------- test that we can read them, should fail
logger.debug("Reading data, should fail this time ");
try {
- readData( "testTypes", entityCount );
+ readData( em, "testTypes", entityCount, 0 );
fail("should have failed to read data");
} catch (Exception expected) {}
- // ----------------- rebuild index
+ // ----------------- rebuild index for catherders only
logger.debug("Preparing to rebuild all indexes");;
@@ -203,7 +218,7 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT {
try {
- setup.getEmf().rebuildApplicationIndexes( em.getApplicationId(), po );
+ setup.getEmf().rebuildCollectionIndex( em.getApplicationId(), "catherders", false, po );
reporter.report();
registry.remove( meterName );
@@ -214,9 +229,10 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT {
fail();
}
- // ----------------- test that we can read them
+ // ----------------- test that we can read the catherder collection and not the catshepard
- readData( "testTypes", entityCount );
+ readData( em, "catherders", herderCount, 0 );
+ readData( em, "catshepards", 0, 0 );
}
@@ -225,7 +241,10 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT {
logger.info("Started rebuildIndex()");
- final EntityManager em = app.getEntityManager();
+ String rand = RandomStringUtils.randomAlphanumeric(5);
+ final UUID appId = setup.createApplication("org_" + rand, "app_" + rand);
+
+ final EntityManager em = setup.getEmf().getEntityManager(appId);
// ----------------- create a bunch of entities
@@ -288,19 +307,19 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT {
// ----------------- test that we can read them, should work fine
logger.debug("Read the data");
- readData("testTypes", entityCount );
+ readData( em, "testTypes", entityCount, 3 );
// ----------------- delete the system and application indexes
logger.debug("Deleting app index and system app index");
- deleteIndex( CpNamingUtils.SYSTEM_APP_ID );
+ //deleteIndex( CpNamingUtils.SYSTEM_APP_ID );
deleteIndex( em.getApplicationId() );
// ----------------- test that we can read them, should fail
logger.debug("Reading data, should fail this time ");
try {
- readData( "testTypes", entityCount );
+ readData( em, "testTypes", entityCount, 3 );
fail("should have failed to read data");
} catch (Exception expected) {}
@@ -349,7 +368,7 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT {
// ----------------- test that we can read them
- readData( "testTypes", entityCount );
+ readData( em, "testTypes", entityCount, 3 );
}
/**
@@ -369,9 +388,9 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT {
}
- private int readData( String collectionName, int expected ) throws Exception {
+ private int readData( EntityManager em,
+ String collectionName, int expectedEntities, int expectedConnections ) throws Exception {
- EntityManager em = app.getEntityManager();
em.refreshIndex();
Query q = Query.fromQL("select * where key1=1000");
@@ -387,7 +406,7 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT {
Results catResults = em.searchConnectedEntities(e,
Query.fromQL("select *").setConnectionType( "herds" ));
- assertEquals( 3, catResults.size() );
+ assertEquals( expectedConnections, catResults.size() );
if ( count % 100 == 0 ) {
logger.info( "read {} entities", count);
@@ -405,9 +424,9 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT {
}
}
- if ( expected != -1 && expected != count ) {
+ if ( expectedEntities != -1 && expectedEntities != count ) {
throw new RuntimeException("Did not get expected "
- + expected + " entities, instead got " + count );
+ + expectedEntities + " entities, instead got " + count );
}
return count;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2f077f1b/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportCollectionIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportCollectionIT.java b/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportCollectionIT.java
index e566512..7fcfc57 100644
--- a/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportCollectionIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportCollectionIT.java
@@ -110,7 +110,8 @@ public class ImportCollectionIT {
bucketPrefix = System.getProperty( "bucketName" );
// start the scheduler after we're all set up
- JobSchedulerService jobScheduler = ConcurrentProcessSingleton.getInstance().getSpringResource().getBean( JobSchedulerService.class );
+ JobSchedulerService jobScheduler = ConcurrentProcessSingleton.getInstance()
+ .getSpringResource().getBean( JobSchedulerService.class );
if ( jobScheduler.state() != Service.State.RUNNING ) {
jobScheduler.startAsync();
jobScheduler.awaitRunning();
@@ -481,7 +482,8 @@ public class ImportCollectionIT {
ImportService importService = setup.getImportService();
- Import importEntity = importService.schedule(em.getApplication().getUuid(), new HashMap<String, Object>() {{
+ Import importEntity = importService.schedule(em.getApplication().getUuid(),
+ new HashMap<String, Object>() {{
put( "path", organization.getName() + em.getApplication().getName() );
put( "organizationId", organization.getUuid() );
put( "applicationId", em.getApplication().getUuid() );
[2/5] incubator-usergrid git commit: Partial implementation of
re-index collection with option to do it in reverse.
Posted by to...@apache.org.
Partial implementation of re-index collection with option to do it in reverse.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/a704ed73
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/a704ed73
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/a704ed73
Branch: refs/heads/USERGRID-273
Commit: a704ed73b8a2b17334ce87b8581d4fe09e63321f
Parents: 1399849
Author: Dave Johnson <dm...@apigee.com>
Authored: Fri Feb 20 12:10:38 2015 -0500
Committer: Dave Johnson <dm...@apigee.com>
Committed: Fri Feb 20 12:10:38 2015 -0500
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 41 ++++--
.../corepersistence/CpEntityManagerFactory.java | 18 ++-
.../usergrid/corepersistence/CpWalker.java | 88 ++++++------
.../usergrid/persistence/EntityManager.java | 3 +
.../persistence/EntityManagerFactory.java | 7 +-
.../cassandra/EntityManagerImpl.java | 5 +
.../PerformanceEntityRebuildIndexTest.java | 136 ++++++++++++++++++-
.../org/apache/usergrid/rest/IndexResource.java | 43 ++++--
.../apache/usergrid/rest/IndexResourceIT.java | 2 +-
9 files changed, 271 insertions(+), 72 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a704ed73/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 7eb9f94..6af0539 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -32,8 +32,6 @@ import java.util.Set;
import java.util.TreeMap;
import java.util.TreeSet;
import java.util.UUID;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.TimeUnit;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -105,10 +103,6 @@ import org.apache.usergrid.utils.StringUtils;
import org.apache.usergrid.utils.UUIDUtils;
import com.google.common.base.Preconditions;
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheLoader;
-import com.google.common.cache.CacheLoader.InvalidCacheLoadException;
-import com.google.common.cache.LoadingCache;
import com.netflix.hystrix.exception.HystrixRuntimeException;
import com.yammer.metrics.annotation.Metered;
@@ -150,7 +144,6 @@ import static org.apache.usergrid.persistence.Schema.PROPERTY_TYPE;
import static org.apache.usergrid.persistence.Schema.PROPERTY_UUID;
import static org.apache.usergrid.persistence.Schema.TYPE_APPLICATION;
import static org.apache.usergrid.persistence.Schema.TYPE_ENTITY;
-import static org.apache.usergrid.persistence.SimpleEntityRef.getUuid;
import static org.apache.usergrid.persistence.cassandra.ApplicationCF.APPLICATION_AGGREGATE_COUNTERS;
import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_COMPOSITE_DICTIONARIES;
import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_COUNTERS;
@@ -2746,6 +2739,38 @@ public class CpEntityManager implements EntityManager {
}
+
+ /**
+ * Completely reindex the named collection in the application associated with this EntityManager.
+ */
+ @Override
+ public void reindexCollection(
+ final EntityManagerFactory.ProgressObserver po, String collectionName) throws Exception {
+
+ CpWalker walker = new CpWalker( );
+
+ walker.walkCollections( this, application, collectionName, new CpVisitor() {
+
+ @Override
+ public void visitCollectionEntry( EntityManager em, String collName, Entity entity ) {
+
+ try {
+ em.update( entity );
+ po.onProgress( entity );
+ }
+ catch ( WriteOptimisticVerifyException wo ) {
+ //swallow this, it just means this was already updated, which accomplishes our task. Just ignore.
+ logger.warn( "Someone beat us to updating entity {} in collection {}. Ignoring.", entity.getName(),
+ collName );
+ }
+ catch ( Exception ex ) {
+ logger.error( "Error repersisting entity", ex );
+ }
+ }
+ } );
+ }
+
+
/**
* Completely reindex the application associated with this EntityManager.
*/
@@ -2753,7 +2778,7 @@ public class CpEntityManager implements EntityManager {
CpWalker walker = new CpWalker( );
- walker.walkCollections( this, application, new CpVisitor() {
+ walker.walkCollections( this, application, null, new CpVisitor() {
@Override
public void visitCollectionEntry( EntityManager em, String collName, Entity entity ) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a704ed73/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index bf85b45..01691d4 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -597,7 +597,8 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
public long performEntityCount() {
//TODO, this really needs to be a task that writes this data somewhere since this will get
//progressively slower as the system expands
- return AllEntitiesInSystemObservable.getAllEntitiesInSystem( managerCache, 1000 ).longCount().toBlocking().last();
+ return AllEntitiesInSystemObservable
+ .getAllEntitiesInSystem( managerCache, 1000 ).longCount().toBlocking().last();
}
@@ -744,8 +745,19 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
}
@Override
- public void rebuildCollectionIndex(UUID appId, String collection, ProgressObserver po ) {
- throw new UnsupportedOperationException( "Not supported yet." );
+ public void rebuildCollectionIndex(
+ UUID appId, String collectionName, boolean reverse, ProgressObserver po ) throws Exception {
+
+ EntityManager em = getEntityManager( appId );
+
+ //explicitly invoke create index, we don't know if it exists or not in ES during a rebuild.
+ em.createIndex();
+ Application app = em.getApplication();
+
+ em.reindexCollection( po, collectionName, reverse );
+
+ logger.info("\n\nRebuilt index for application {} id {} collection {}\n",
+ new Object[] { app.getName(), appId, collectionName } );
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a704ed73/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpWalker.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpWalker.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpWalker.java
index 8a9eed5..6bd90ec 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpWalker.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpWalker.java
@@ -55,15 +55,18 @@ public class CpWalker {
}
- public void walkCollections( final CpEntityManager em, final EntityRef start,
- final CpVisitor visitor ) throws Exception {
+ public void walkCollections(final CpEntityManager em, final EntityRef start,
+ String collectionName, final CpVisitor visitor) throws Exception {
- doWalkCollections( em, new SimpleId( start.getUuid(), start.getType() ), visitor );
+ doWalkCollections( em, collectionName, new SimpleId( start.getUuid(), start.getType() ), visitor );
}
private void doWalkCollections(
- final CpEntityManager em, final Id applicationId, final CpVisitor visitor ) {
+ final CpEntityManager em,
+ final String collectionName,
+ final Id applicationId,
+ final CpVisitor visitor ) {
final ApplicationScope applicationScope = em.getApplicationScope();
@@ -77,10 +80,17 @@ public class CpWalker {
applicationScope.getApplication().getUuid()
} );
- //only search edge types that start with collections
+ // only search edge types that start with collections
+
+ final String edgeType;
+ if ( collectionName != null ) {
+ edgeType = CpNamingUtils.EDGE_COLL_SUFFIX;
+ } else {
+ edgeType = CpNamingUtils.getEdgeTypeFromCollectionName( collectionName );
+ }
Observable<String> edgeTypes = gm.getEdgeTypesFromSource(
- new SimpleSearchEdgeType( applicationId, CpNamingUtils.EDGE_COLL_SUFFIX, null ) );
+ new SimpleSearchEdgeType( applicationId, edgeType, null ) );
edgeTypes.flatMap( new Func1<String, Observable<Edge>>() {
@Override
@@ -88,45 +98,45 @@ public class CpWalker {
logger.debug( "Loading edges of type {} from node {}", edgeType, applicationId );
- return gm.loadEdgesFromSource( new SimpleSearchByEdgeType( applicationId, edgeType, Long.MAX_VALUE,
- SearchByEdgeType.Order.DESCENDING, null ) );
- }
- } )
- //process our edges in parallel for as much efficiency as possible
- .parallel( new Func1<Observable<Edge>, Observable<Edge>>() {
- @Override
- public Observable<Edge> call( final Observable<Edge> edgeObservable ) {
- //visit and update the entity
- return edgeObservable.doOnNext( new Action1<Edge>() {
-
- @Override
- public void call( Edge edge ) {
-
- logger.info( "Re-indexing edge {}", edge );
-
- EntityRef targetNodeEntityRef =
- new SimpleEntityRef( edge.getTargetNode().getType(), edge.getTargetNode().getUuid() );
-
- Entity entity;
- try {
- entity = em.get( targetNodeEntityRef );
- }
- catch ( Exception ex ) {
- logger.error( "Error getting sourceEntity {}:{}, continuing", targetNodeEntityRef.getType(),
- targetNodeEntityRef.getUuid() );
- return;
- }
+ return gm.loadEdgesFromSource( new SimpleSearchByEdgeType(
+ applicationId, edgeType, Long.MAX_VALUE, SearchByEdgeType.Order.DESCENDING, null ) );
+ }
- String collName = CpNamingUtils.getCollectionName( edge.getType() );
+ } ).parallel( new Func1<Observable<Edge>, Observable<Edge>>() {
- visitor.visitCollectionEntry( em, collName, entity );
- }
- } );
+ @Override
+ public Observable<Edge> call( final Observable<Edge> edgeObservable ) { // process edges in parallel
+ return edgeObservable.doOnNext( new Action1<Edge>() { // visit and update then entity
+
+ @Override
+ public void call( Edge edge ) {
+
+ logger.info( "Re-indexing edge {}", edge );
+
+ EntityRef targetNodeEntityRef = new SimpleEntityRef(
+ edge.getTargetNode().getType(),
+ edge.getTargetNode().getUuid() );
+
+ Entity entity;
+ try {
+ entity = em.get( targetNodeEntityRef );
+ }
+ catch ( Exception ex ) {
+ logger.error( "Error getting sourceEntity {}:{}, continuing",
+ targetNodeEntityRef.getType(),
+ targetNodeEntityRef.getUuid() );
+ return;
+ }
+
+ String collName = CpNamingUtils.getCollectionName( edge.getType() );
+ visitor.visitCollectionEntry( em, collName, entity );
+ }
+ } );
}
}, Schedulers.io() )
- //wait for it to complete
+ // wait for it to complete
.toBlocking().lastOrDefault( null ); // end foreach on edges
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a704ed73/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
index d425b7a..b7ca477 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
@@ -703,6 +703,9 @@ public interface EntityManager {
/** For testing purposes */
public void flushManagerCaches();
+ void reindexCollection(
+ EntityManagerFactory.ProgressObserver po, String collectionName) throws Exception;
+
public void reindex( final EntityManagerFactory.ProgressObserver po ) throws Exception;
/**
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a704ed73/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
index 7367eca..6a9095d 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
@@ -130,32 +130,29 @@ public interface EntityManagerFactory {
/**
* Return the migration status message
- * @return
*/
public String getMigrateDataStatus();
/**
* Return the current migration version of the system
- * @return
*/
public int getMigrateDataVersion();
/**
* Force the migration version to the specified version
- * @param version
*/
public void setMigrationVersion(int version);
/**
* Perform a realtime count of every entity in the system. This can be slow as it traverses the entire system graph
- * @return
*/
public long performEntityCount();
/** For testing purposes */
public void flushEntityManagerCaches();
- public void rebuildCollectionIndex(UUID appId, String collection, ProgressObserver object);
+ void rebuildCollectionIndex(
+ UUID appId, String collection, boolean reverse, ProgressObserver po) throws Exception;
/**
* Add a new index to the application for scale
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a704ed73/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
index 6c940a7..d0fe985 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
@@ -2932,6 +2932,11 @@ public class EntityManagerImpl implements EntityManager {
}
@Override
+ public void reindexCollection(EntityManagerFactory.ProgressObserver po, String collectionName) throws Exception {
+ throw new UnsupportedOperationException("Not supported.");
+ }
+
+ @Override
public void reindex(EntityManagerFactory.ProgressObserver po) throws Exception {
throw new UnsupportedOperationException("Not supported.");
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a704ed73/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java b/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
index f8eec52..e853e94 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
@@ -88,6 +88,139 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT {
@Test
+ public void rebuildOneCollectionIndex() throws Exception {
+
+ logger.info("Started rebuildIndex()");
+
+ final EntityManager em = app.getEntityManager();
+
+ // ----------------- create a bunch of entities
+
+ Map<String, Object> entityMap = new HashMap<String, Object>() {{
+ put("key1", 1000 );
+ put("key2", 2000 );
+ put("key3", "Some value");
+ }};
+ Map<String, Object> cat1map = new HashMap<String, Object>() {{
+ put("name", "enzo");
+ put("color", "orange");
+ }};
+ Map<String, Object> cat2map = new HashMap<String, Object>() {{
+ put("name", "marquee");
+ put("color", "grey");
+ }};
+ Map<String, Object> cat3map = new HashMap<String, Object>() {{
+ put("name", "bertha");
+ put("color", "tabby");
+ }};
+
+ Entity cat1 = em.create("cat", cat1map );
+ Entity cat2 = em.create("cat", cat2map );
+ Entity cat3 = em.create("cat", cat3map );
+
+ final long stopTime = System.currentTimeMillis() + RUNTIME_MS;
+
+ List<EntityRef> entityRefs = new ArrayList<EntityRef>();
+ int entityCount = 0;
+ while ( System.currentTimeMillis() < stopTime ) {
+
+ final Entity entity;
+
+ try {
+ entityMap.put("key", entityCount );
+ entity = em.create("testType", entityMap );
+
+ em.refreshIndex();
+
+ em.createConnection(entity, "herds", cat1);
+ em.createConnection(entity, "herds", cat2);
+ em.createConnection(entity, "herds", cat3);
+
+ } catch (Exception ex) {
+ throw new RuntimeException("Error creating entity", ex);
+ }
+
+ entityRefs.add(new SimpleEntityRef( entity.getType(), entity.getUuid() ) );
+ if ( entityCount % 10 == 0 ) {
+ logger.info("Created {} entities", entityCount );
+ }
+
+ entityCount++;
+ try { Thread.sleep( WRITE_DELAY_MS ); } catch (InterruptedException ignored ) {}
+ }
+
+ logger.info("Created {} entities", entityCount);
+ em.refreshIndex();
+
+ // ----------------- test that we can read them, should work fine
+
+ logger.debug("Read the data");
+ readData("testTypes", entityCount );
+
+ // ----------------- delete the system and application indexes
+
+ logger.debug("Deleting app index and system app index");
+ deleteIndex( CpNamingUtils.SYSTEM_APP_ID );
+ deleteIndex( em.getApplicationId() );
+
+ // ----------------- test that we can read them, should fail
+
+ logger.debug("Reading data, should fail this time ");
+ try {
+ readData( "testTypes", entityCount );
+ fail("should have failed to read data");
+
+ } catch (Exception expected) {}
+
+ // ----------------- rebuild index
+
+ logger.debug("Preparing to rebuild all indexes");;
+
+ final String meterName = this.getClass().getSimpleName() + ".rebuildIndex";
+ final Meter meter = registry.meter( meterName );
+
+ EntityManagerFactory.ProgressObserver po = new EntityManagerFactory.ProgressObserver() {
+ int counter = 0;
+
+ @Override
+ public void onProgress( final EntityRef entity ) {
+
+ meter.mark();
+ logger.debug("Indexing {}:{}", entity.getType(), entity.getUuid());
+ if ( !logger.isDebugEnabled() && counter % 100 == 0 ) {
+ logger.info("Reindexed {} entities", counter );
+ }
+ counter++;
+ }
+
+
+
+ @Override
+ public long getWriteDelayTime() {
+ return 0;
+ }
+ };
+
+ try {
+
+ setup.getEmf().rebuildApplicationIndexes( em.getApplicationId(), po );
+
+ reporter.report();
+ registry.remove( meterName );
+ logger.info("Rebuilt index");
+
+ } catch (Exception ex) {
+ logger.error("Error rebuilding index", ex);
+ fail();
+ }
+
+ // ----------------- test that we can read them
+
+ readData( "testTypes", entityCount );
+ }
+
+
+ @Test
public void rebuildIndex() throws Exception {
logger.info("Started rebuildIndex()");
@@ -252,7 +385,8 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT {
assertEquals( 2000, e.getProperty("key2"));
- Results catResults = em.searchConnectedEntities(e, Query.fromQL("select *").setConnectionType( "herds" ));
+ Results catResults = em.searchConnectedEntities(e,
+ Query.fromQL("select *").setConnectionType( "herds" ));
assertEquals( 3, catResults.size() );
if ( count % 100 == 0 ) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a704ed73/stack/rest/src/main/java/org/apache/usergrid/rest/IndexResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/IndexResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/IndexResource.java
index 7b391a1..251e25c 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/IndexResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/IndexResource.java
@@ -160,12 +160,13 @@ public class IndexResource extends AbstractContextResource {
@RequireSystemAccess
@PUT
@Path( "rebuild/" + RootResource.APPLICATION_ID_PATH + "/{collectionName}" )
- public JSONWithPadding rebuildIndexes( @Context UriInfo ui,
- @PathParam( "applicationId" ) final String applicationIdStr,
- @PathParam( "collectionName" ) final String collectionName,
- @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback,
- @QueryParam( "delay" ) @DefaultValue( "10" ) final long delay )
- throws Exception {
+ public JSONWithPadding rebuildIndexes(
+ @Context UriInfo ui,
+ @PathParam( "applicationId" ) final String applicationIdStr,
+ @PathParam( "collectionName" ) final String collectionName,
+ @QueryParam( "reverse" ) @DefaultValue( "false" ) final Boolean reverse,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback,
+ @QueryParam( "delay" ) @DefaultValue( "10" ) final long delay ) throws Exception {
final UUID appId = UUIDUtils.tryExtractUUID( applicationIdStr );
ApiResponse response = createApiResponse();
@@ -175,7 +176,13 @@ public class IndexResource extends AbstractContextResource {
public void run() {
- rebuildCollection( appId, collectionName, delay );
+ try {
+ rebuildCollection( appId, collectionName, reverse, delay );
+ } catch (Exception e) {
+
+ // TODO: handle this in rebuildCollection() instead
+ throw new RuntimeException("Error rebuilding collection");
+ }
}
};
@@ -192,13 +199,14 @@ public class IndexResource extends AbstractContextResource {
@POST
@Path( RootResource.APPLICATION_ID_PATH )
public JSONWithPadding addIndex(@Context UriInfo ui,
- @PathParam( "applicationId" ) final String applicationIdStr,
- Map<String, Object> config,
- @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback) throws Exception{
+ @PathParam( "applicationId" ) final String applicationIdStr,
+ Map<String, Object> config,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback) throws Exception{
+
ApiResponse response = createApiResponse();
final UUID appId = UUIDUtils.tryExtractUUID(applicationIdStr);
- if(!config.containsKey("replicas") || !config.containsKey("shards") ||
+ if (!config.containsKey("replicas") || !config.containsKey("shards") ||
!(config.get("replicas") instanceof Integer) || !(config.get("shards") instanceof Integer)){
throw new IllegalArgumentException("body must contains 'replicas' of type int and 'shards' of type int");
}
@@ -207,14 +215,20 @@ public class IndexResource extends AbstractContextResource {
throw new IllegalArgumentException("Please add an indexSuffix to your post");
}
- emf.addIndex(appId, config.get("indexSuffix").toString(), (int) config.get("shards"),(int) config.get("replicas"));
+ emf.addIndex(appId, config.get("indexSuffix").toString(),
+ (int) config.get("shards"),(int) config.get("replicas"));
response.setAction("Add index to alias");
return new JSONWithPadding(response, callback);
}
- private void rebuildCollection( final UUID applicationId, final String collectionName, final long delay ) {
+ private void rebuildCollection(
+ final UUID applicationId,
+ final String collectionName,
+ final boolean reverse,
+ final long delay ) throws Exception {
+
EntityManagerFactory.ProgressObserver po = new EntityManagerFactory.ProgressObserver() {
@Override
@@ -227,10 +241,9 @@ public class IndexResource extends AbstractContextResource {
}
};
-
logger.info( "Reindexing for app id: {} and collection {}", applicationId, collectionName );
- emf.rebuildCollectionIndex( applicationId, collectionName, po );
+ emf.rebuildCollectionIndex(applicationId, collectionName, reverse, po);
emf.refreshIndex();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a704ed73/stack/rest/src/test/java/org/apache/usergrid/rest/IndexResourceIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/IndexResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/IndexResourceIT.java
index 9bf0813..7f8d085 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/IndexResourceIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/IndexResourceIT.java
@@ -68,7 +68,7 @@ public class IndexResourceIT extends AbstractRestIT {
.queryParam("access_token", superToken)
.accept(MediaType.APPLICATION_JSON).type(MediaType.APPLICATION_JSON_TYPE)
.post(String.class, data));
- }catch (Exception e){
+ } catch (Exception e) {
LOG.error("failed", e);
fail(e.toString());
}