You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sn...@apache.org on 2014/09/26 18:38:09 UTC
[1/4] git commit: Remove redundant dependencies.
Repository: incubator-usergrid
Updated Branches:
refs/heads/two-dot-o 9fd7a89a5 -> 261d9e850
Remove redundant dependencies.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/8a1dafff
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/8a1dafff
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/8a1dafff
Branch: refs/heads/two-dot-o
Commit: 8a1dafff715c82ccc25c0b6fcfad529de8083a7b
Parents: 9fd7a89
Author: Dave Johnson <dm...@apigee.com>
Authored: Fri Sep 26 12:34:13 2014 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Fri Sep 26 12:34:13 2014 -0400
----------------------------------------------------------------------
stack/core/pom.xml | 10 ----------
1 file changed, 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8a1dafff/stack/core/pom.xml
----------------------------------------------------------------------
diff --git a/stack/core/pom.xml b/stack/core/pom.xml
index 383a3fa..6cd5dac 100644
--- a/stack/core/pom.xml
+++ b/stack/core/pom.xml
@@ -583,16 +583,6 @@
<version>2.7.0</version>
</dependency>
- <dependency>
- <groupId>com.codahale.metrics</groupId>
- <artifactId>metrics-core</artifactId>
- <version>${metrics.version}</version>
- </dependency>
- <dependency>
- <groupId>com.codahale.metrics</groupId>
- <artifactId>metrics-graphite</artifactId>
- <version>${metrics.version}</version>
- </dependency>
</dependencies>
</project>
[2/4] git commit: Added notes about how to setup log4j logging.
Posted by sn...@apache.org.
Added notes about how to setup log4j logging.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/121fd148
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/121fd148
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/121fd148
Branch: refs/heads/two-dot-o
Commit: 121fd148baf46ace3cb216e9672d33719c282eb4
Parents: 8a1daff
Author: Dave Johnson <dm...@apigee.com>
Authored: Fri Sep 26 12:35:02 2014 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Fri Sep 26 12:35:02 2014 -0400
----------------------------------------------------------------------
stack/tools/README.md | 18 ++++++++++++++++++
1 file changed, 18 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/121fd148/stack/tools/README.md
----------------------------------------------------------------------
diff --git a/stack/tools/README.md b/stack/tools/README.md
index f8fcd71..e03fc35 100644
--- a/stack/tools/README.md
+++ b/stack/tools/README.md
@@ -47,6 +47,24 @@ What Tools are available?
---
This README.md only documents two of the tools, WarehouseExport and WarehouseUpsert. You will have to seek documentation else where or look at the source code (in the org.apache.usergrid.tools package) to understand what other tools are avialable.
+Enabling additional logging for debugging purposes
+---
+
+If you want to control what is logged by the Usergrid Tools, you must provide your own log4j.configuration file. When you start the tool with the java command, add the following option right after the word "java" and (of course) replace /path/to with the real path to your log4j file.
+
+ -Dlog4j.configuration=file:/path/to/log4j.properties
+
+Here is a simple log4j.properties file that will turn on DEBUG logging for the Usergrid Tools:
+
+ log4j.rootLogger=WARN,stdout
+ log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+ log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+ log4j.appender.stdout.layout.ConversionPattern=%d %p (%t) [%c] - %m%n
+
+ log4j.category.org.apache.usergrid.tools=DEBUG
+ log4j.category.org.apache.usergrid=INFO
+ log4j.logger.org.apache.usergrid.persistence.cassandra=WARN
+
Redshift Warehouse Export and Upsert
===
[3/4] git commit: Should be doing update() here in update().
Posted by sn...@apache.org.
Should be doing update() here in update().
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/368264f8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/368264f8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/368264f8
Branch: refs/heads/two-dot-o
Commit: 368264f8839284a42d556c28ea02a3e91a392a53
Parents: 121fd14
Author: Dave Johnson <dm...@apigee.com>
Authored: Fri Sep 26 12:35:35 2014 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Fri Sep 26 12:35:35 2014 -0400
----------------------------------------------------------------------
.../java/org/apache/usergrid/corepersistence/CpEntityManager.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/368264f8/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index f906167..5a5e6bc 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -503,7 +503,7 @@ public class CpEntityManager implements EntityManager {
logger.debug("About to Write {}:{} version {}", new Object[] {
cpEntity.getId().getType(), cpEntity.getId().getUuid(), cpEntity.getVersion() });
- cpEntity = ecm.write( cpEntity ).toBlockingObservable().last();
+ cpEntity = ecm.update( cpEntity ).toBlockingObservable().last();
logger.debug("Wrote {}:{} version {}", new Object[] {
cpEntity.getId().getType(), cpEntity.getId().getUuid(), cpEntity.getVersion() });
[4/4] git commit: Updating the IndexTool to work with two-dot-o and
Core Persistence.
Posted by sn...@apache.org.
Updating the IndexTool to work with two-dot-o and Core Persistence.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/261d9e85
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/261d9e85
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/261d9e85
Branch: refs/heads/two-dot-o
Commit: 261d9e850e92ad33eb4b39cca7fca133716ac252
Parents: 368264f
Author: Dave Johnson <dm...@apigee.com>
Authored: Fri Sep 26 12:37:04 2014 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Fri Sep 26 12:37:04 2014 -0400
----------------------------------------------------------------------
.../corepersistence/CpEntityManagerFactory.java | 184 +++++++++++++++++--
.../corepersistence/CpRelationManager.java | 14 +-
.../HybridEntityManagerFactory.java | 10 +
.../persistence/EntityManagerFactory.java | 4 +
.../cassandra/EntityManagerFactoryImpl.java | 51 +++++
.../cassandra/QueryProcessorTest.java | 2 -
.../org/apache/usergrid/tools/IndexRebuild.java | 93 +++-------
.../org/apache/usergrid/tools/ToolBase.java | 26 +--
8 files changed, 279 insertions(+), 105 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/261d9e85/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index f9fe945..3e9a6a9 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -24,12 +24,14 @@ import static java.lang.String.CASE_INSENSITIVE_ORDER;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
+import java.util.Set;
import java.util.TreeMap;
import java.util.UUID;
import org.apache.commons.lang.StringUtils;
import org.apache.usergrid.persistence.DynamicEntity;
import org.apache.usergrid.persistence.EntityManager;
import org.apache.usergrid.persistence.EntityManagerFactory;
+import org.apache.usergrid.persistence.Results;
import static org.apache.usergrid.persistence.Schema.PROPERTY_CREATED;
import static org.apache.usergrid.persistence.Schema.PROPERTY_NAME;
import static org.apache.usergrid.persistence.Schema.PROPERTY_UUID;
@@ -45,6 +47,7 @@ import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
import org.apache.usergrid.persistence.entities.Application;
import org.apache.usergrid.persistence.exceptions.ApplicationAlreadyExistsException;
+import org.apache.usergrid.persistence.exceptions.DuplicateUniquePropertyExistsException;
import org.apache.usergrid.persistence.graph.GraphManagerFactory;
import org.apache.usergrid.persistence.index.EntityIndex;
import org.apache.usergrid.persistence.index.EntityIndexFactory;
@@ -93,7 +96,6 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
public static final UUID DEFAULT_APPLICATION_ID =
UUID.fromString("b6768a08-b5d5-11e3-a495-11ddb1de66c9");
-
// Three types of things we store in System Application
public static final String SYSTEM_APPS_TYPE = "zzzappszzz";
public static final String SYSTEM_ORGS_TYPE = "zzzorgszzz";
@@ -102,9 +104,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
private static final Id systemAppId =
new SimpleId( UUID.fromString(SYSTEM_APPS_UUID), SYSTEM_APPS_TYPE );
- // Scopes for those three types of things
-
- public static final CollectionScope SYSTEM_APP_SCOPE =
+ public static final CollectionScope SYSTEM_APPS_SCOPE =
new CollectionScopeImpl( systemAppId, systemAppId, SYSTEM_APPS_TYPE );
public static final IndexScope SYSTEM_APPS_INDEX_SCOPE =
new IndexScopeImpl( systemAppId, systemAppId, SYSTEM_APPS_TYPE);
@@ -136,6 +136,8 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
private boolean skipAggregateCounters;
+ private static final int REBUILD_PAGE_SIZE = 100;
+
public CpEntityManagerFactory(
CassandraService cass, CounterUtils counterUtils, boolean skipAggregateCounters ) {
@@ -281,7 +283,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
// create app in system app scope
{
EntityCollectionManager ecm = getManagerCache()
- .getEntityCollectionManager( SYSTEM_APP_SCOPE );
+ .getEntityCollectionManager(SYSTEM_APPS_SCOPE );
EntityIndex eci = getManagerCache()
.getEntityIndex( SYSTEM_APPS_INDEX_SCOPE );
@@ -354,24 +356,45 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
@Metered(group = "core", name = "EntityManagerFactory_getApplication")
public Map<String, UUID> getApplications() throws Exception {
- Query q = Query.fromQL("select *");
-
EntityCollectionManager em = getManagerCache()
- .getEntityCollectionManager( SYSTEM_APP_SCOPE );
+ .getEntityCollectionManager(SYSTEM_APPS_SCOPE );
EntityIndex ei = getManagerCache()
.getEntityIndex( SYSTEM_APPS_INDEX_SCOPE );
- CandidateResults results = ei.search( q );
-
Map<String, UUID> appMap = new HashMap<String, UUID>();
- Iterator<CandidateResult> iter = results.iterator();
- while ( iter.hasNext() ) {
- CandidateResult cr = iter.next();
- Entity e = em.load( cr.getId() ).toBlockingObservable().last();
- appMap.put(
- (String)e.getField(PROPERTY_NAME).getValue(),
- (UUID)e.getField(PROPERTY_UUID).getValue() );
+ String cursor = null;
+ boolean done = false;
+
+ while ( !done ) {
+
+ Query q = Query.fromQL("select *");
+ q.setCursor( cursor );
+
+ CandidateResults results = ei.search( q );
+ cursor = results.getCursor();
+
+ Iterator<CandidateResult> iter = results.iterator();
+ while ( iter.hasNext() ) {
+
+ CandidateResult cr = iter.next();
+ Entity e = em.load( cr.getId() ).toBlockingObservable().last();
+
+ if ( cr.getVersion().compareTo( e.getVersion()) < 0 ) {
+ logger.debug("Stale version of Entity uuid:{} type:{}, stale v:{}, latest v:{}",
+ new Object[] { cr.getId().getUuid(), cr.getId().getType(),
+ cr.getVersion(), e.getVersion()});
+ continue;
+ }
+
+ appMap.put(
+ (String)e.getField(PROPERTY_NAME).getValue(),
+ (UUID)e.getField("applicationUuid").getValue() );
+ }
+
+ if ( cursor == null ) {
+ done = true;
+ }
}
return appMap;
@@ -435,7 +458,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
}
// intentionally going only one-level deep into fields and treating all
- // values as strings because that is all we need for service properties.'
+ // values as strings because that is all we need for service properties
for ( String key : properties.keySet() ) {
propsEntity.setField( new StringField(key, properties.get(key)) );
}
@@ -557,6 +580,131 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
managerCache.getEntityIndex( dscope ).refresh();
}
+
+ public void rebuildInternalIndexes() throws Exception {
+
+ logger.info("Rebuilding system apps index");
+ rebuildIndexScope(
+ CpEntityManagerFactory.SYSTEM_APPS_SCOPE,
+ CpEntityManagerFactory.SYSTEM_APPS_INDEX_SCOPE );
+
+ logger.info("Rebuilding system orgs index");
+ rebuildIndexScope(
+ CpEntityManagerFactory.SYSTEM_ORGS_SCOPE,
+ CpEntityManagerFactory.SYSTEM_ORGS_INDEX_SCOPE );
+
+ logger.info("Rebuilding system props index");
+ rebuildIndexScope(
+ CpEntityManagerFactory.SYSTEM_PROPS_SCOPE,
+ CpEntityManagerFactory.SYSTEM_PROPS_INDEX_SCOPE );
+
+ logger.info("Rebuilding management application index");
+ rebuildApplicationIndex( MANAGEMENT_APPLICATION_ID );
+
+ logger.info("Rebuilding default application index");
+ rebuildApplicationIndex( DEFAULT_APPLICATION_ID );
+ }
+
+
+ private void rebuildIndexScope( CollectionScope cs, IndexScope is ) {
+
+ logger.info("Rebuild index scope for {}:{}:{}", new Object[] {
+ cs.getOwner(), cs.getApplication(), cs.getName()
+ });
+
+ EntityCollectionManager ecm = managerCache.getEntityCollectionManager( cs );
+ EntityIndex ei = managerCache.getEntityIndex( is );
+
+ Query q = Query.fromQL("select *");
+ CandidateResults results = ei.search( q );
+
+ Map<String, UUID> appMap = new HashMap<String, UUID>();
+
+ Iterator<CandidateResult> iter = results.iterator();
+ while (iter.hasNext()) {
+ CandidateResult cr = iter.next();
+
+ Entity entity = ecm.load(cr.getId()).toBlockingObservable().last();
+
+ if (cr.getVersion().compareTo( entity.getVersion()) < 0 ) {
+ logger.warn(" Ignoring stale version uuid:{} type:{} version:{} latest version:{}",
+ new Object[]{
+ cr.getId().getUuid(),
+ cr.getId().getType(),
+ cr.getVersion(),
+ entity.getVersion()
+ });
+ continue;
+ }
+
+ logger.info(" Updating CP Entity type: {} with id: {} for app id: {}",
+ new Object[]{cr.getId().getType(), cr.getId().getUuid(),
+ CpEntityManagerFactory.SYSTEM_APPS_SCOPE.getApplication().getUuid()
+ }
+ );
+
+ ei.index(entity);
+ }
+
+ }
+
+
+ public void rebuildApplicationIndex( UUID appId ) throws Exception {
+
+ EntityManager em = getEntityManager( appId );
+
+ Set<String> collections = em.getApplicationCollections();
+
+ logger.debug("For app {} found {} collections: {}", new Object[] {
+ appId, collections.size(), collections });
+
+ for ( String collection : collections ) {
+ rebuildCollectionIndex( appId, collection );
+ }
+ }
+
+
+ public void rebuildCollectionIndex( UUID appId, String collectionName ) throws Exception {
+
+ logger.info( "Reindexing collection: {} for app id: {}", collectionName, appId );
+
+ EntityManager em = getEntityManager( appId );
+ Application app = em.getApplication();
+
+ // search for all orgs
+
+ Query query = new Query();
+ query.setLimit(REBUILD_PAGE_SIZE );
+ Results r = null;
+
+ do {
+
+ r = em.searchCollection( app, collectionName, query );
+
+ for ( org.apache.usergrid.persistence.Entity entity : r.getEntities() ) {
+
+ logger.info( " Updating Entity name {}, type: {}, id: {} in app id: {}", new Object[] {
+ entity.getName(), entity.getType(), entity.getUuid(), appId
+ } );
+
+ try {
+ em.update( entity );
+ }
+ catch ( DuplicateUniquePropertyExistsException dupee ) {
+ logger.error( "Duplicate property for type: {} with id: {} for app id: {}. "
+ + "Property name: {} , value: {}", new Object[] {
+ entity.getType(), entity.getUuid(), appId, dupee.getPropertyName(),
+ dupee.getPropertyValue()
+ } );
+ }
+ }
+
+ query.setCursor( r.getCursor() );
+ }
+ while ( r != null && r.size() == REBUILD_PAGE_SIZE );
+ }
+
+
@Override
public void flushEntityManagerCaches() {
Map<UUID, EntityManager> entityManagersMap = entityManagers.asMap();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/261d9e85/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index c2f4a09..0345e23 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -586,11 +586,13 @@ public class CpRelationManager implements RelationManager {
GraphManager gm = managerCache.getGraphManager(applicationScope);
- Observable<String> str = gm.getEdgeTypesFromSource( new SimpleSearchEdgeType( cpHeadEntity.getId(),null , null ));
+ Observable<String> str = gm.getEdgeTypesFromSource(
+ new SimpleSearchEdgeType( cpHeadEntity.getId(),null , null ));
Iterator<String> iter = str.toBlockingObservable().getIterator();
while ( iter.hasNext() ) {
- indexes.add( iter.next() );
+ String edgeType = iter.next();
+ indexes.add( getCollectionName( edgeType ) );
}
return indexes;
@@ -1695,10 +1697,10 @@ public class CpRelationManager implements RelationManager {
}
if ( cr.getVersion().compareTo( e.getVersion()) < 0 ) {
- logger.debug("Stale version uuid:{} type:{} version:{} latest version:{}",
- new Object[] {cr.getId().getUuid(), cr.getId().getType(), cr.getVersion(),
- e.getVersion() });
- continue;
+ logger.debug("Stale version of Entity uuid:{} type:{}, stale v:{}, latest v:{}",
+ new Object[] { cr.getId().getUuid(), cr.getId().getType(),
+ cr.getVersion(), e.getVersion()});
+ continue;
}
org.apache.usergrid.persistence.model.entity.Entity alreadySeen =
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/261d9e85/stack/core/src/main/java/org/apache/usergrid/corepersistence/HybridEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/HybridEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/HybridEntityManagerFactory.java
index 8284a0e..3b2fd3e 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/HybridEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/HybridEntityManagerFactory.java
@@ -143,4 +143,14 @@ public class HybridEntityManagerFactory implements EntityManagerFactory, Applica
public void flushEntityManagerCaches() {
factory.flushEntityManagerCaches();
}
+
+ @Override
+ public void rebuildInternalIndexes() throws Exception {
+ factory.rebuildInternalIndexes();
+ }
+
+ @Override
+ public void rebuildCollectionIndex(UUID appId, String collectionName) throws Exception {
+ factory.rebuildCollectionIndex(appId, collectionName);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/261d9e85/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
index 6a2d30f..15e4dbe 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
@@ -117,6 +117,10 @@ public interface EntityManagerFactory {
public void refreshIndex();
+ public void rebuildInternalIndexes() throws Exception;
+
+ public void rebuildCollectionIndex( UUID appId, String collectionName ) throws Exception;
+
public void setApplicationContext(ApplicationContext ac);
/** For testing purposes */
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/261d9e85/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
index 5d090f5..d1a98ec 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
@@ -56,6 +56,7 @@ import static java.lang.String.CASE_INSENSITIVE_ORDER;
import static me.prettyprint.hector.api.factory.HFactory.createRangeSlicesQuery;
+import org.apache.usergrid.persistence.Results;
import static org.apache.usergrid.persistence.Schema.PROPERTY_NAME;
import static org.apache.usergrid.persistence.Schema.PROPERTY_UUID;
import static org.apache.usergrid.persistence.Schema.TYPE_APPLICATION;
@@ -67,6 +68,8 @@ import static org.apache.usergrid.persistence.cassandra.CassandraService.PROPERT
import static org.apache.usergrid.persistence.cassandra.CassandraService.RETRY_COUNT;
import static org.apache.usergrid.utils.ConversionUtils.uuid;
import static org.apache.usergrid.persistence.cassandra.Serializers.*;
+import org.apache.usergrid.persistence.exceptions.DuplicateUniquePropertyExistsException;
+import org.apache.usergrid.persistence.index.query.Query;
/**
@@ -97,6 +100,8 @@ public class EntityManagerFactoryImpl implements EntityManagerFactory, Applicati
}
} );
+ private static final int REBUILD_PAGE_SIZE = 100;
+
/**
* Must be constructed with a CassandraClientPool.
@@ -417,4 +422,50 @@ public class EntityManagerFactoryImpl implements EntityManagerFactory, Applicati
// no-op
}
+ @Override
+ public void rebuildInternalIndexes() throws Exception {
+ throw new UnsupportedOperationException("Not supported yet.");
+ }
+
+ @Override
+ public void rebuildCollectionIndex(UUID appId, String collectionName) throws Exception {
+
+ logger.info( "Reindexing collection: {} for app id: {}", collectionName, appId );
+
+ EntityManager em = getEntityManager( appId );
+ Application app = em.getApplication();
+
+ // search for all orgs
+
+ Query query = new Query();
+ query.setLimit(REBUILD_PAGE_SIZE );
+ Results r = null;
+
+ do {
+
+ r = em.searchCollection( app, collectionName, query );
+
+ for ( org.apache.usergrid.persistence.Entity entity : r.getEntities() ) {
+ logger.info( "Updating entity type: {} with id: {} for app id: {}", new Object[] {
+ entity.getType(), entity.getUuid(), appId
+ } );
+
+ try {
+ em.update( entity );
+ }
+ catch ( DuplicateUniquePropertyExistsException dupee ) {
+ logger.error( "duplicate property for type: {} with id: {} for app id: {}. "
+ + "Property name: {} , value: {}", new Object[] {
+ entity.getType(), entity.getUuid(), appId, dupee.getPropertyName(),
+ dupee.getPropertyValue()
+ } );
+ }
+ }
+
+ query.setCursor( r.getCursor() );
+ }
+ while ( r != null && r.size() == REBUILD_PAGE_SIZE );
+
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/261d9e85/stack/core/src/test/java/org/apache/usergrid/persistence/cassandra/QueryProcessorTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/cassandra/QueryProcessorTest.java b/stack/core/src/test/java/org/apache/usergrid/persistence/cassandra/QueryProcessorTest.java
index c527eb8..fe04d23 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/cassandra/QueryProcessorTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/cassandra/QueryProcessorTest.java
@@ -35,8 +35,6 @@ import org.apache.usergrid.persistence.index.query.tree.CpQueryFilterParser;
import org.apache.usergrid.persistence.query.ir.AndNode;
import org.apache.usergrid.persistence.query.ir.NotNode;
import org.apache.usergrid.persistence.query.ir.OrNode;
-import org.apache.usergrid.persistence.query.ir.OrderByNode;
-import org.apache.usergrid.persistence.query.ir.QueryNode;
import org.apache.usergrid.persistence.query.ir.QuerySlice;
import org.apache.usergrid.persistence.query.ir.SliceNode;
import org.apache.usergrid.persistence.query.ir.WithinNode;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/261d9e85/stack/tools/src/main/java/org/apache/usergrid/tools/IndexRebuild.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/IndexRebuild.java b/stack/tools/src/main/java/org/apache/usergrid/tools/IndexRebuild.java
index 0a8b5cb..40aa0a2 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/IndexRebuild.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/IndexRebuild.java
@@ -26,12 +26,7 @@ import java.util.UUID;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.usergrid.persistence.Entity;
import org.apache.usergrid.persistence.EntityManager;
-import org.apache.usergrid.persistence.index.query.Query;
-import org.apache.usergrid.persistence.Results;
-import org.apache.usergrid.persistence.entities.Application;
-import org.apache.usergrid.persistence.exceptions.DuplicateUniquePropertyExistsException;
import org.apache.usergrid.utils.UUIDUtils;
import org.apache.commons.cli.CommandLine;
@@ -42,26 +37,15 @@ import org.apache.commons.cli.Options;
/**
- * This is a utility to load all entities in an application and re-save them, this forces the secondary indexing to be
- * updated.
- *
- * @author tnine
+ * This is a utility to load all entities in an application and re-save them, this forces
+ * the secondary indexing to be updated.
*/
public class IndexRebuild extends ToolBase {
- /**
- *
- */
private static final String APPLICATION_ARG = "app";
- /**
- *
- */
private static final String COLLECTION_ARG = "col";
- /**
- *
- */
private static final int PAGE_SIZE = 100;
@@ -72,21 +56,27 @@ public class IndexRebuild extends ToolBase {
@SuppressWarnings("static-access")
public Options createOptions() {
- Option hostOption =
- OptionBuilder.withArgName( "host" ).hasArg().isRequired( true ).withDescription( "Cassandra host" )
- .create( "host" );
+ Option hostOpt = OptionBuilder.withArgName( "host" ).hasArg().isRequired( true )
+ .withDescription( "Cassandra host" ).create( "host" );
+
+ Option esHostsOpt = OptionBuilder.withArgName( "host" ).hasArg().isRequired( true )
+ .withDescription( "ElasticSearch host" ).create( "eshost" );
- Option appOption = OptionBuilder.withArgName( APPLICATION_ARG ).hasArg().isRequired( false )
- .withDescription( "application id or app name" ).create( APPLICATION_ARG );
+ Option esClusterOpt = OptionBuilder.withArgName( "host" ).hasArg().isRequired( true )
+ .withDescription( "ElasticSearch cluster name" ).create( "escluster" );
- Option collectionOption = OptionBuilder.withArgName( COLLECTION_ARG ).hasArg().isRequired( false )
- .withDescription( "colleciton name" ).create( COLLECTION_ARG );
+ Option appOpt = OptionBuilder.withArgName( APPLICATION_ARG ).hasArg().isRequired( false )
+ .withDescription( "Application id or app name" ).create( APPLICATION_ARG );
+ Option collOpt = OptionBuilder.withArgName( COLLECTION_ARG ).hasArg().isRequired( false )
+ .withDescription( "Collection name" ).create( COLLECTION_ARG );
Options options = new Options();
- options.addOption( hostOption );
- options.addOption( appOption );
- options.addOption( collectionOption );
+ options.addOption( hostOpt );
+ options.addOption( esHostsOpt );
+ options.addOption( esClusterOpt );
+ options.addOption( appOpt );
+ options.addOption( collOpt );
return options;
}
@@ -104,18 +94,20 @@ public class IndexRebuild extends ToolBase {
logger.info( "Starting index rebuild" );
+ emf.rebuildInternalIndexes();
+ emf.refreshIndex();
+
/**
* Goes through each app id specified
*/
for ( UUID appId : getAppIds( line ) ) {
logger.info( "Reindexing for app id: {}", appId );
-
Set<String> collections = getCollections( line, appId );
for ( String collection : collections ) {
-
reindex( appId, collection );
+ emf.refreshIndex();
}
}
@@ -127,21 +119,21 @@ public class IndexRebuild extends ToolBase {
private Collection<UUID> getAppIds( CommandLine line ) throws Exception {
String appId = line.getOptionValue( APPLICATION_ARG );
+ Map<String, UUID> ids = emf.getApplications();
+
if ( appId != null ) {
UUID id = UUIDUtils.tryExtractUUID( appId );
if ( id == null ) {
+ logger.debug("Got applications: " + ids );
id = emf.getApplications().get( appId );
}
return Collections.singleton( id );
}
- Map<String, UUID> ids = emf.getApplications();
-
System.out.println( "Printing all apps" );
-
for ( Entry<String, UUID> entry : ids.entrySet() ) {
System.out.println( entry.getKey() );
}
@@ -167,39 +159,6 @@ public class IndexRebuild extends ToolBase {
/** The application id. The collection name. */
private void reindex( UUID appId, String collectionName ) throws Exception {
- logger.info( "Reindexing collection: {} for app id: {}", collectionName, appId );
-
- EntityManager em = emf.getEntityManager( appId );
- Application app = em.getApplication();
-
- // search for all orgs
-
- Query query = new Query();
- query.setLimit( PAGE_SIZE );
- Results r = null;
-
- do {
-
- r = em.searchCollection( app, collectionName, query );
-
- for ( Entity entity : r.getEntities() ) {
- logger.info( "Updating entity type: {} with id: {} for app id: {}", new Object[] {
- entity.getType(), entity.getUuid(), appId
- } );
-
- try {
- em.update( entity );
- }
- catch ( DuplicateUniquePropertyExistsException dupee ) {
- logger.error( "duplicate property for type: {} with id: {} for app id: {}. Property name: {} , "
- + "value: {}", new Object[] {
- entity.getType(), entity.getUuid(), appId, dupee.getPropertyName(), dupee.getPropertyValue()
- } );
- }
- }
-
- query.setCursor( r.getCursor() );
- }
- while ( r != null && r.size() == PAGE_SIZE );
+ emf.rebuildCollectionIndex(appId, collectionName);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/261d9e85/stack/tools/src/main/java/org/apache/usergrid/tools/ToolBase.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/ToolBase.java b/stack/tools/src/main/java/org/apache/usergrid/tools/ToolBase.java
index 4d19afa..d40434f 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/ToolBase.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/ToolBase.java
@@ -50,8 +50,8 @@ import static org.apache.usergrid.utils.JsonUtils.mapToFormattedJsonString;
/**
- * Base class for Usergrid Tools commands. Any class that implements this can be called with java -jar {jarname}
- * org.apache.usergrid.tools.{classname}.
+ * Base class for Usergrid Tools commands. Any class that implements this can be called with
+ * java -jar {jarname} org.apache.usergrid.tools.{classname}.
*/
public abstract class ToolBase {
@@ -95,6 +95,8 @@ public abstract class ToolBase {
if ( line.hasOption( "host" ) ) {
System.setProperty( "cassandra.url", line.getOptionValue( "host" ) );
+ System.setProperty( "elasticsearch.hosts", line.getOptionValue( "eshost" ) );
+ System.setProperty( "elasticsearch.cluster_name", line.getOptionValue( "escluster" ) );
}
try {
@@ -110,7 +112,7 @@ public abstract class ToolBase {
public void printCliHelp( String message ) {
System.out.println( message );
HelpFormatter formatter = new HelpFormatter();
- formatter.printHelp( "java -jar usergrid-tools-0.0.1-SNAPSHOT.jar " + getToolName(), createOptions() );
+ formatter.printHelp( "java -jar usergrid-tools.jar " + getToolName(), createOptions() );
System.exit( -1 );
}
@@ -123,14 +125,15 @@ public abstract class ToolBase {
@SuppressWarnings("static-access")
public Options createOptions() {
- Option hostOption =
- OptionBuilder.withArgName( "host" ).hasArg().withDescription( "Cassandra host" ).create( "host" );
+ Option hostOption = OptionBuilder.withArgName( "host" ).hasArg()
+ .withDescription( "Cassandra host" ).create( "host" );
- Option remoteOption = OptionBuilder.withDescription( "Use remote Cassandra instance" ).create( "remote" );
+ Option remoteOption = OptionBuilder
+ .withDescription( "Use remote Cassandra instance" ).create( "remote" );
- Option verbose =
- OptionBuilder.withDescription( "Print on the console an echo of the content written to the file" )
- .create( VERBOSE );
+ Option verbose = OptionBuilder
+ .withDescription( "Print on the console an echo of the content written to the file" )
+ .create( VERBOSE );
Options options = new Options();
options.addOption( hostOption );
@@ -155,7 +158,6 @@ public abstract class ToolBase {
public void startSpring() {
- // copy("/testApplicationContext.xml", TMP);
String[] locations = { "toolsApplicationContext.xml" };
ApplicationContext ac = new ClassPathXmlApplicationContext( locations );
@@ -164,8 +166,8 @@ public abstract class ToolBase {
acbf.initializeBean( this, "testClient" );
assertNotNull( emf );
- assertTrue( "EntityManagerFactory is instance of EntityManagerFactoryImpl",
- emf instanceof EntityManagerFactoryImpl );
+ assertTrue( "EntityManagerFactory is instance of EntityManagerFactory",
+ emf instanceof EntityManagerFactory );
}