You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by to...@apache.org on 2014/09/29 22:53:07 UTC
[28/52] [abbrv] git commit: Updating the IndexTool to work with
two-dot-o and Core Persistence.
Updating the IndexTool to work with two-dot-o and Core Persistence.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/261d9e85
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/261d9e85
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/261d9e85
Branch: refs/heads/eventsystem
Commit: 261d9e850e92ad33eb4b39cca7fca133716ac252
Parents: 368264f
Author: Dave Johnson <dm...@apigee.com>
Authored: Fri Sep 26 12:37:04 2014 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Fri Sep 26 12:37:04 2014 -0400
----------------------------------------------------------------------
.../corepersistence/CpEntityManagerFactory.java | 184 +++++++++++++++++--
.../corepersistence/CpRelationManager.java | 14 +-
.../HybridEntityManagerFactory.java | 10 +
.../persistence/EntityManagerFactory.java | 4 +
.../cassandra/EntityManagerFactoryImpl.java | 51 +++++
.../cassandra/QueryProcessorTest.java | 2 -
.../org/apache/usergrid/tools/IndexRebuild.java | 93 +++-------
.../org/apache/usergrid/tools/ToolBase.java | 26 +--
8 files changed, 279 insertions(+), 105 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/261d9e85/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index f9fe945..3e9a6a9 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -24,12 +24,14 @@ import static java.lang.String.CASE_INSENSITIVE_ORDER;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
+import java.util.Set;
import java.util.TreeMap;
import java.util.UUID;
import org.apache.commons.lang.StringUtils;
import org.apache.usergrid.persistence.DynamicEntity;
import org.apache.usergrid.persistence.EntityManager;
import org.apache.usergrid.persistence.EntityManagerFactory;
+import org.apache.usergrid.persistence.Results;
import static org.apache.usergrid.persistence.Schema.PROPERTY_CREATED;
import static org.apache.usergrid.persistence.Schema.PROPERTY_NAME;
import static org.apache.usergrid.persistence.Schema.PROPERTY_UUID;
@@ -45,6 +47,7 @@ import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
import org.apache.usergrid.persistence.entities.Application;
import org.apache.usergrid.persistence.exceptions.ApplicationAlreadyExistsException;
+import org.apache.usergrid.persistence.exceptions.DuplicateUniquePropertyExistsException;
import org.apache.usergrid.persistence.graph.GraphManagerFactory;
import org.apache.usergrid.persistence.index.EntityIndex;
import org.apache.usergrid.persistence.index.EntityIndexFactory;
@@ -93,7 +96,6 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
public static final UUID DEFAULT_APPLICATION_ID =
UUID.fromString("b6768a08-b5d5-11e3-a495-11ddb1de66c9");
-
// Three types of things we store in System Application
public static final String SYSTEM_APPS_TYPE = "zzzappszzz";
public static final String SYSTEM_ORGS_TYPE = "zzzorgszzz";
@@ -102,9 +104,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
private static final Id systemAppId =
new SimpleId( UUID.fromString(SYSTEM_APPS_UUID), SYSTEM_APPS_TYPE );
- // Scopes for those three types of things
-
- public static final CollectionScope SYSTEM_APP_SCOPE =
+ public static final CollectionScope SYSTEM_APPS_SCOPE =
new CollectionScopeImpl( systemAppId, systemAppId, SYSTEM_APPS_TYPE );
public static final IndexScope SYSTEM_APPS_INDEX_SCOPE =
new IndexScopeImpl( systemAppId, systemAppId, SYSTEM_APPS_TYPE);
@@ -136,6 +136,8 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
private boolean skipAggregateCounters;
+ private static final int REBUILD_PAGE_SIZE = 100;
+
public CpEntityManagerFactory(
CassandraService cass, CounterUtils counterUtils, boolean skipAggregateCounters ) {
@@ -281,7 +283,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
// create app in system app scope
{
EntityCollectionManager ecm = getManagerCache()
- .getEntityCollectionManager( SYSTEM_APP_SCOPE );
+ .getEntityCollectionManager(SYSTEM_APPS_SCOPE );
EntityIndex eci = getManagerCache()
.getEntityIndex( SYSTEM_APPS_INDEX_SCOPE );
@@ -354,24 +356,45 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
@Metered(group = "core", name = "EntityManagerFactory_getApplication")
public Map<String, UUID> getApplications() throws Exception {
- Query q = Query.fromQL("select *");
-
EntityCollectionManager em = getManagerCache()
- .getEntityCollectionManager( SYSTEM_APP_SCOPE );
+ .getEntityCollectionManager(SYSTEM_APPS_SCOPE );
EntityIndex ei = getManagerCache()
.getEntityIndex( SYSTEM_APPS_INDEX_SCOPE );
- CandidateResults results = ei.search( q );
-
Map<String, UUID> appMap = new HashMap<String, UUID>();
- Iterator<CandidateResult> iter = results.iterator();
- while ( iter.hasNext() ) {
- CandidateResult cr = iter.next();
- Entity e = em.load( cr.getId() ).toBlockingObservable().last();
- appMap.put(
- (String)e.getField(PROPERTY_NAME).getValue(),
- (UUID)e.getField(PROPERTY_UUID).getValue() );
+ String cursor = null;
+ boolean done = false;
+
+ while ( !done ) {
+
+ Query q = Query.fromQL("select *");
+ q.setCursor( cursor );
+
+ CandidateResults results = ei.search( q );
+ cursor = results.getCursor();
+
+ Iterator<CandidateResult> iter = results.iterator();
+ while ( iter.hasNext() ) {
+
+ CandidateResult cr = iter.next();
+ Entity e = em.load( cr.getId() ).toBlockingObservable().last();
+
+ if ( cr.getVersion().compareTo( e.getVersion()) < 0 ) {
+ logger.debug("Stale version of Entity uuid:{} type:{}, stale v:{}, latest v:{}",
+ new Object[] { cr.getId().getUuid(), cr.getId().getType(),
+ cr.getVersion(), e.getVersion()});
+ continue;
+ }
+
+ appMap.put(
+ (String)e.getField(PROPERTY_NAME).getValue(),
+ (UUID)e.getField("applicationUuid").getValue() );
+ }
+
+ if ( cursor == null ) {
+ done = true;
+ }
}
return appMap;
@@ -435,7 +458,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
}
// intentionally going only one-level deep into fields and treating all
- // values as strings because that is all we need for service properties.'
+ // values as strings because that is all we need for service properties
for ( String key : properties.keySet() ) {
propsEntity.setField( new StringField(key, properties.get(key)) );
}
@@ -557,6 +580,131 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
managerCache.getEntityIndex( dscope ).refresh();
}
+
+ public void rebuildInternalIndexes() throws Exception {
+
+ logger.info("Rebuilding system apps index");
+ rebuildIndexScope(
+ CpEntityManagerFactory.SYSTEM_APPS_SCOPE,
+ CpEntityManagerFactory.SYSTEM_APPS_INDEX_SCOPE );
+
+ logger.info("Rebuilding system orgs index");
+ rebuildIndexScope(
+ CpEntityManagerFactory.SYSTEM_ORGS_SCOPE,
+ CpEntityManagerFactory.SYSTEM_ORGS_INDEX_SCOPE );
+
+ logger.info("Rebuilding system props index");
+ rebuildIndexScope(
+ CpEntityManagerFactory.SYSTEM_PROPS_SCOPE,
+ CpEntityManagerFactory.SYSTEM_PROPS_INDEX_SCOPE );
+
+ logger.info("Rebuilding management application index");
+ rebuildApplicationIndex( MANAGEMENT_APPLICATION_ID );
+
+ logger.info("Rebuilding default application index");
+ rebuildApplicationIndex( DEFAULT_APPLICATION_ID );
+ }
+
+
+ private void rebuildIndexScope( CollectionScope cs, IndexScope is ) {
+
+ logger.info("Rebuild index scope for {}:{}:{}", new Object[] {
+ cs.getOwner(), cs.getApplication(), cs.getName()
+ });
+
+ EntityCollectionManager ecm = managerCache.getEntityCollectionManager( cs );
+ EntityIndex ei = managerCache.getEntityIndex( is );
+
+ Query q = Query.fromQL("select *");
+ CandidateResults results = ei.search( q );
+
+ Map<String, UUID> appMap = new HashMap<String, UUID>();
+
+ Iterator<CandidateResult> iter = results.iterator();
+ while (iter.hasNext()) {
+ CandidateResult cr = iter.next();
+
+ Entity entity = ecm.load(cr.getId()).toBlockingObservable().last();
+
+ if (cr.getVersion().compareTo( entity.getVersion()) < 0 ) {
+ logger.warn(" Ignoring stale version uuid:{} type:{} version:{} latest version:{}",
+ new Object[]{
+ cr.getId().getUuid(),
+ cr.getId().getType(),
+ cr.getVersion(),
+ entity.getVersion()
+ });
+ continue;
+ }
+
+ logger.info(" Updating CP Entity type: {} with id: {} for app id: {}",
+ new Object[]{cr.getId().getType(), cr.getId().getUuid(),
+ CpEntityManagerFactory.SYSTEM_APPS_SCOPE.getApplication().getUuid()
+ }
+ );
+
+ ei.index(entity);
+ }
+
+ }
+
+
+ public void rebuildApplicationIndex( UUID appId ) throws Exception {
+
+ EntityManager em = getEntityManager( appId );
+
+ Set<String> collections = em.getApplicationCollections();
+
+ logger.debug("For app {} found {} collections: {}", new Object[] {
+ appId, collections.size(), collections });
+
+ for ( String collection : collections ) {
+ rebuildCollectionIndex( appId, collection );
+ }
+ }
+
+
+ public void rebuildCollectionIndex( UUID appId, String collectionName ) throws Exception {
+
+ logger.info( "Reindexing collection: {} for app id: {}", collectionName, appId );
+
+ EntityManager em = getEntityManager( appId );
+ Application app = em.getApplication();
+
+ // search for all orgs
+
+ Query query = new Query();
+ query.setLimit(REBUILD_PAGE_SIZE );
+ Results r = null;
+
+ do {
+
+ r = em.searchCollection( app, collectionName, query );
+
+ for ( org.apache.usergrid.persistence.Entity entity : r.getEntities() ) {
+
+ logger.info( " Updating Entity name {}, type: {}, id: {} in app id: {}", new Object[] {
+ entity.getName(), entity.getType(), entity.getUuid(), appId
+ } );
+
+ try {
+ em.update( entity );
+ }
+ catch ( DuplicateUniquePropertyExistsException dupee ) {
+ logger.error( "Duplicate property for type: {} with id: {} for app id: {}. "
+ + "Property name: {} , value: {}", new Object[] {
+ entity.getType(), entity.getUuid(), appId, dupee.getPropertyName(),
+ dupee.getPropertyValue()
+ } );
+ }
+ }
+
+ query.setCursor( r.getCursor() );
+ }
+ while ( r != null && r.size() == REBUILD_PAGE_SIZE );
+ }
+
+
@Override
public void flushEntityManagerCaches() {
Map<UUID, EntityManager> entityManagersMap = entityManagers.asMap();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/261d9e85/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index c2f4a09..0345e23 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -586,11 +586,13 @@ public class CpRelationManager implements RelationManager {
GraphManager gm = managerCache.getGraphManager(applicationScope);
- Observable<String> str = gm.getEdgeTypesFromSource( new SimpleSearchEdgeType( cpHeadEntity.getId(),null , null ));
+ Observable<String> str = gm.getEdgeTypesFromSource(
+ new SimpleSearchEdgeType( cpHeadEntity.getId(),null , null ));
Iterator<String> iter = str.toBlockingObservable().getIterator();
while ( iter.hasNext() ) {
- indexes.add( iter.next() );
+ String edgeType = iter.next();
+ indexes.add( getCollectionName( edgeType ) );
}
return indexes;
@@ -1695,10 +1697,10 @@ public class CpRelationManager implements RelationManager {
}
if ( cr.getVersion().compareTo( e.getVersion()) < 0 ) {
- logger.debug("Stale version uuid:{} type:{} version:{} latest version:{}",
- new Object[] {cr.getId().getUuid(), cr.getId().getType(), cr.getVersion(),
- e.getVersion() });
- continue;
+ logger.debug("Stale version of Entity uuid:{} type:{}, stale v:{}, latest v:{}",
+ new Object[] { cr.getId().getUuid(), cr.getId().getType(),
+ cr.getVersion(), e.getVersion()});
+ continue;
}
org.apache.usergrid.persistence.model.entity.Entity alreadySeen =
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/261d9e85/stack/core/src/main/java/org/apache/usergrid/corepersistence/HybridEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/HybridEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/HybridEntityManagerFactory.java
index 8284a0e..3b2fd3e 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/HybridEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/HybridEntityManagerFactory.java
@@ -143,4 +143,14 @@ public class HybridEntityManagerFactory implements EntityManagerFactory, Applica
public void flushEntityManagerCaches() {
factory.flushEntityManagerCaches();
}
+
+ @Override
+ public void rebuildInternalIndexes() throws Exception {
+ factory.rebuildInternalIndexes();
+ }
+
+ @Override
+ public void rebuildCollectionIndex(UUID appId, String collectionName) throws Exception {
+ factory.rebuildCollectionIndex(appId, collectionName);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/261d9e85/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
index 6a2d30f..15e4dbe 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
@@ -117,6 +117,10 @@ public interface EntityManagerFactory {
public void refreshIndex();
+ public void rebuildInternalIndexes() throws Exception;
+
+ public void rebuildCollectionIndex( UUID appId, String collectionName ) throws Exception;
+
public void setApplicationContext(ApplicationContext ac);
/** For testing purposes */
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/261d9e85/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
index 5d090f5..d1a98ec 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
@@ -56,6 +56,7 @@ import static java.lang.String.CASE_INSENSITIVE_ORDER;
import static me.prettyprint.hector.api.factory.HFactory.createRangeSlicesQuery;
+import org.apache.usergrid.persistence.Results;
import static org.apache.usergrid.persistence.Schema.PROPERTY_NAME;
import static org.apache.usergrid.persistence.Schema.PROPERTY_UUID;
import static org.apache.usergrid.persistence.Schema.TYPE_APPLICATION;
@@ -67,6 +68,8 @@ import static org.apache.usergrid.persistence.cassandra.CassandraService.PROPERT
import static org.apache.usergrid.persistence.cassandra.CassandraService.RETRY_COUNT;
import static org.apache.usergrid.utils.ConversionUtils.uuid;
import static org.apache.usergrid.persistence.cassandra.Serializers.*;
+import org.apache.usergrid.persistence.exceptions.DuplicateUniquePropertyExistsException;
+import org.apache.usergrid.persistence.index.query.Query;
/**
@@ -97,6 +100,8 @@ public class EntityManagerFactoryImpl implements EntityManagerFactory, Applicati
}
} );
+ private static final int REBUILD_PAGE_SIZE = 100;
+
/**
* Must be constructed with a CassandraClientPool.
@@ -417,4 +422,50 @@ public class EntityManagerFactoryImpl implements EntityManagerFactory, Applicati
// no-op
}
+ @Override
+ public void rebuildInternalIndexes() throws Exception {
+ throw new UnsupportedOperationException("Not supported yet.");
+ }
+
+ @Override
+ public void rebuildCollectionIndex(UUID appId, String collectionName) throws Exception {
+
+ logger.info( "Reindexing collection: {} for app id: {}", collectionName, appId );
+
+ EntityManager em = getEntityManager( appId );
+ Application app = em.getApplication();
+
+ // search for all orgs
+
+ Query query = new Query();
+ query.setLimit(REBUILD_PAGE_SIZE );
+ Results r = null;
+
+ do {
+
+ r = em.searchCollection( app, collectionName, query );
+
+ for ( org.apache.usergrid.persistence.Entity entity : r.getEntities() ) {
+ logger.info( "Updating entity type: {} with id: {} for app id: {}", new Object[] {
+ entity.getType(), entity.getUuid(), appId
+ } );
+
+ try {
+ em.update( entity );
+ }
+ catch ( DuplicateUniquePropertyExistsException dupee ) {
+ logger.error( "duplicate property for type: {} with id: {} for app id: {}. "
+ + "Property name: {} , value: {}", new Object[] {
+ entity.getType(), entity.getUuid(), appId, dupee.getPropertyName(),
+ dupee.getPropertyValue()
+ } );
+ }
+ }
+
+ query.setCursor( r.getCursor() );
+ }
+ while ( r != null && r.size() == REBUILD_PAGE_SIZE );
+
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/261d9e85/stack/core/src/test/java/org/apache/usergrid/persistence/cassandra/QueryProcessorTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/cassandra/QueryProcessorTest.java b/stack/core/src/test/java/org/apache/usergrid/persistence/cassandra/QueryProcessorTest.java
index c527eb8..fe04d23 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/cassandra/QueryProcessorTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/cassandra/QueryProcessorTest.java
@@ -35,8 +35,6 @@ import org.apache.usergrid.persistence.index.query.tree.CpQueryFilterParser;
import org.apache.usergrid.persistence.query.ir.AndNode;
import org.apache.usergrid.persistence.query.ir.NotNode;
import org.apache.usergrid.persistence.query.ir.OrNode;
-import org.apache.usergrid.persistence.query.ir.OrderByNode;
-import org.apache.usergrid.persistence.query.ir.QueryNode;
import org.apache.usergrid.persistence.query.ir.QuerySlice;
import org.apache.usergrid.persistence.query.ir.SliceNode;
import org.apache.usergrid.persistence.query.ir.WithinNode;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/261d9e85/stack/tools/src/main/java/org/apache/usergrid/tools/IndexRebuild.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/IndexRebuild.java b/stack/tools/src/main/java/org/apache/usergrid/tools/IndexRebuild.java
index 0a8b5cb..40aa0a2 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/IndexRebuild.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/IndexRebuild.java
@@ -26,12 +26,7 @@ import java.util.UUID;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.usergrid.persistence.Entity;
import org.apache.usergrid.persistence.EntityManager;
-import org.apache.usergrid.persistence.index.query.Query;
-import org.apache.usergrid.persistence.Results;
-import org.apache.usergrid.persistence.entities.Application;
-import org.apache.usergrid.persistence.exceptions.DuplicateUniquePropertyExistsException;
import org.apache.usergrid.utils.UUIDUtils;
import org.apache.commons.cli.CommandLine;
@@ -42,26 +37,15 @@ import org.apache.commons.cli.Options;
/**
- * This is a utility to load all entities in an application and re-save them, this forces the secondary indexing to be
- * updated.
- *
- * @author tnine
+ * This is a utility to load all entities in an application and re-save them, this forces
+ * the secondary indexing to be updated.
*/
public class IndexRebuild extends ToolBase {
- /**
- *
- */
private static final String APPLICATION_ARG = "app";
- /**
- *
- */
private static final String COLLECTION_ARG = "col";
- /**
- *
- */
private static final int PAGE_SIZE = 100;
@@ -72,21 +56,27 @@ public class IndexRebuild extends ToolBase {
@SuppressWarnings("static-access")
public Options createOptions() {
- Option hostOption =
- OptionBuilder.withArgName( "host" ).hasArg().isRequired( true ).withDescription( "Cassandra host" )
- .create( "host" );
+ Option hostOpt = OptionBuilder.withArgName( "host" ).hasArg().isRequired( true )
+ .withDescription( "Cassandra host" ).create( "host" );
+
+ Option esHostsOpt = OptionBuilder.withArgName( "host" ).hasArg().isRequired( true )
+ .withDescription( "ElasticSearch host" ).create( "eshost" );
- Option appOption = OptionBuilder.withArgName( APPLICATION_ARG ).hasArg().isRequired( false )
- .withDescription( "application id or app name" ).create( APPLICATION_ARG );
+ Option esClusterOpt = OptionBuilder.withArgName( "host" ).hasArg().isRequired( true )
+ .withDescription( "ElasticSearch cluster name" ).create( "escluster" );
- Option collectionOption = OptionBuilder.withArgName( COLLECTION_ARG ).hasArg().isRequired( false )
- .withDescription( "colleciton name" ).create( COLLECTION_ARG );
+ Option appOpt = OptionBuilder.withArgName( APPLICATION_ARG ).hasArg().isRequired( false )
+ .withDescription( "Application id or app name" ).create( APPLICATION_ARG );
+ Option collOpt = OptionBuilder.withArgName( COLLECTION_ARG ).hasArg().isRequired( false )
+ .withDescription( "Collection name" ).create( COLLECTION_ARG );
Options options = new Options();
- options.addOption( hostOption );
- options.addOption( appOption );
- options.addOption( collectionOption );
+ options.addOption( hostOpt );
+ options.addOption( esHostsOpt );
+ options.addOption( esClusterOpt );
+ options.addOption( appOpt );
+ options.addOption( collOpt );
return options;
}
@@ -104,18 +94,20 @@ public class IndexRebuild extends ToolBase {
logger.info( "Starting index rebuild" );
+ emf.rebuildInternalIndexes();
+ emf.refreshIndex();
+
/**
* Goes through each app id specified
*/
for ( UUID appId : getAppIds( line ) ) {
logger.info( "Reindexing for app id: {}", appId );
-
Set<String> collections = getCollections( line, appId );
for ( String collection : collections ) {
-
reindex( appId, collection );
+ emf.refreshIndex();
}
}
@@ -127,21 +119,21 @@ public class IndexRebuild extends ToolBase {
private Collection<UUID> getAppIds( CommandLine line ) throws Exception {
String appId = line.getOptionValue( APPLICATION_ARG );
+ Map<String, UUID> ids = emf.getApplications();
+
if ( appId != null ) {
UUID id = UUIDUtils.tryExtractUUID( appId );
if ( id == null ) {
+ logger.debug("Got applications: " + ids );
id = emf.getApplications().get( appId );
}
return Collections.singleton( id );
}
- Map<String, UUID> ids = emf.getApplications();
-
System.out.println( "Printing all apps" );
-
for ( Entry<String, UUID> entry : ids.entrySet() ) {
System.out.println( entry.getKey() );
}
@@ -167,39 +159,6 @@ public class IndexRebuild extends ToolBase {
/** The application id. The collection name. */
private void reindex( UUID appId, String collectionName ) throws Exception {
- logger.info( "Reindexing collection: {} for app id: {}", collectionName, appId );
-
- EntityManager em = emf.getEntityManager( appId );
- Application app = em.getApplication();
-
- // search for all orgs
-
- Query query = new Query();
- query.setLimit( PAGE_SIZE );
- Results r = null;
-
- do {
-
- r = em.searchCollection( app, collectionName, query );
-
- for ( Entity entity : r.getEntities() ) {
- logger.info( "Updating entity type: {} with id: {} for app id: {}", new Object[] {
- entity.getType(), entity.getUuid(), appId
- } );
-
- try {
- em.update( entity );
- }
- catch ( DuplicateUniquePropertyExistsException dupee ) {
- logger.error( "duplicate property for type: {} with id: {} for app id: {}. Property name: {} , "
- + "value: {}", new Object[] {
- entity.getType(), entity.getUuid(), appId, dupee.getPropertyName(), dupee.getPropertyValue()
- } );
- }
- }
-
- query.setCursor( r.getCursor() );
- }
- while ( r != null && r.size() == PAGE_SIZE );
+ emf.rebuildCollectionIndex(appId, collectionName);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/261d9e85/stack/tools/src/main/java/org/apache/usergrid/tools/ToolBase.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/ToolBase.java b/stack/tools/src/main/java/org/apache/usergrid/tools/ToolBase.java
index 4d19afa..d40434f 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/ToolBase.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/ToolBase.java
@@ -50,8 +50,8 @@ import static org.apache.usergrid.utils.JsonUtils.mapToFormattedJsonString;
/**
- * Base class for Usergrid Tools commands. Any class that implements this can be called with java -jar {jarname}
- * org.apache.usergrid.tools.{classname}.
+ * Base class for Usergrid Tools commands. Any class that implements this can be called with
+ * java -jar {jarname} org.apache.usergrid.tools.{classname}.
*/
public abstract class ToolBase {
@@ -95,6 +95,8 @@ public abstract class ToolBase {
if ( line.hasOption( "host" ) ) {
System.setProperty( "cassandra.url", line.getOptionValue( "host" ) );
+ System.setProperty( "elasticsearch.hosts", line.getOptionValue( "eshost" ) );
+ System.setProperty( "elasticsearch.cluster_name", line.getOptionValue( "escluster" ) );
}
try {
@@ -110,7 +112,7 @@ public abstract class ToolBase {
public void printCliHelp( String message ) {
System.out.println( message );
HelpFormatter formatter = new HelpFormatter();
- formatter.printHelp( "java -jar usergrid-tools-0.0.1-SNAPSHOT.jar " + getToolName(), createOptions() );
+ formatter.printHelp( "java -jar usergrid-tools.jar " + getToolName(), createOptions() );
System.exit( -1 );
}
@@ -123,14 +125,15 @@ public abstract class ToolBase {
@SuppressWarnings("static-access")
public Options createOptions() {
- Option hostOption =
- OptionBuilder.withArgName( "host" ).hasArg().withDescription( "Cassandra host" ).create( "host" );
+ Option hostOption = OptionBuilder.withArgName( "host" ).hasArg()
+ .withDescription( "Cassandra host" ).create( "host" );
- Option remoteOption = OptionBuilder.withDescription( "Use remote Cassandra instance" ).create( "remote" );
+ Option remoteOption = OptionBuilder
+ .withDescription( "Use remote Cassandra instance" ).create( "remote" );
- Option verbose =
- OptionBuilder.withDescription( "Print on the console an echo of the content written to the file" )
- .create( VERBOSE );
+ Option verbose = OptionBuilder
+ .withDescription( "Print on the console an echo of the content written to the file" )
+ .create( VERBOSE );
Options options = new Options();
options.addOption( hostOption );
@@ -155,7 +158,6 @@ public abstract class ToolBase {
public void startSpring() {
- // copy("/testApplicationContext.xml", TMP);
String[] locations = { "toolsApplicationContext.xml" };
ApplicationContext ac = new ClassPathXmlApplicationContext( locations );
@@ -164,8 +166,8 @@ public abstract class ToolBase {
acbf.initializeBean( this, "testClient" );
assertNotNull( emf );
- assertTrue( "EntityManagerFactory is instance of EntityManagerFactoryImpl",
- emf instanceof EntityManagerFactoryImpl );
+ assertTrue( "EntityManagerFactory is instance of EntityManagerFactory",
+ emf instanceof EntityManagerFactory );
}