You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sn...@apache.org on 2015/11/06 21:37:55 UTC
[01/50] [abbrv] usergrid git commit: Merge commit
'refs/pull/413/head' of github.com:apache/usergrid into 2.1-release
Repository: usergrid
Updated Branches:
refs/heads/asf-site 0fd9939bc -> cbe501afa
Merge commit 'refs/pull/413/head' of github.com:apache/usergrid into 2.1-release
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/2aec8a0f
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/2aec8a0f
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/2aec8a0f
Branch: refs/heads/asf-site
Commit: 2aec8a0fa3e607c67dafd5a8e041674ac9859237
Parents: 07d2ad3 752bf4e
Author: Todd Nine <tn...@apigee.com>
Authored: Mon Oct 26 18:39:27 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Mon Oct 26 18:39:27 2015 -0600
----------------------------------------------------------------------
.../organizations/OrganizationsResource.java | 42 +++++++++++++-------
1 file changed, 28 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
[44/50] [abbrv] usergrid git commit: This closes #46
Posted by sn...@apache.org.
This closes #46
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/02d6a914
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/02d6a914
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/02d6a914
Branch: refs/heads/asf-site
Commit: 02d6a914d85f9ef4d631c166164b59b9f1650acc
Parents: 568d35d
Author: Mike Dunker <md...@apigee.com>
Authored: Tue Nov 3 14:49:43 2015 -0800
Committer: Mike Dunker <md...@apigee.com>
Committed: Tue Nov 3 14:49:43 2015 -0800
----------------------------------------------------------------------
----------------------------------------------------------------------
[34/50] [abbrv] usergrid git commit: Remove unnecessary debug print
statement.
Posted by sn...@apache.org.
Remove unnecessary debug print statement.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/8737f69d
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/8737f69d
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/8737f69d
Branch: refs/heads/asf-site
Commit: 8737f69d3b78847e5ef40d0f23fec800aec0dcae
Parents: 9ada6ea
Author: Dave Johnson <sn...@apache.org>
Authored: Fri Oct 30 08:31:00 2015 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Fri Oct 30 08:31:00 2015 -0400
----------------------------------------------------------------------
stack/scripts/multitenant_migrate.py | 2 --
1 file changed, 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/8737f69d/stack/scripts/multitenant_migrate.py
----------------------------------------------------------------------
diff --git a/stack/scripts/multitenant_migrate.py b/stack/scripts/multitenant_migrate.py
index 62e46af..6955de7 100644
--- a/stack/scripts/multitenant_migrate.py
+++ b/stack/scripts/multitenant_migrate.py
@@ -657,8 +657,6 @@ class Migrate:
for appId in apps.values():
app_ids.append(appId)
- print app_ids
-
return app_ids
except requests.exceptions.RequestException as e:
[20/50] [abbrv] usergrid git commit: -Convert search in ES to use
term queries instead of scroll with filters. -Fix delete so we remove
connections from Elasticsearch as well as the entity on entity delete event.
Posted by sn...@apache.org.
-Convert search in ES to use term queries instead of scroll with filters.
-Fix delete so we remove connections from Elasticsearch as well as the entity on entity delete event.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/22beca2c
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/22beca2c
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/22beca2c
Branch: refs/heads/asf-site
Commit: 22beca2cb3714833ddf6a748522c6bead17fe9f2
Parents: 2686054
Author: Michael Russo <mi...@gmail.com>
Authored: Wed Oct 28 15:58:41 2015 -0700
Committer: Michael Russo <mi...@gmail.com>
Committed: Wed Oct 28 15:58:41 2015 -0700
----------------------------------------------------------------------
.../corepersistence/index/IndexServiceImpl.java | 2 +-
.../persistence/index/CandidateResult.java | 11 +-
.../persistence/index/EntityIndexBatch.java | 2 +-
.../usergrid/persistence/index/IndexFig.java | 6 +
.../index/impl/DeIndexOperation.java | 5 +
.../index/impl/EsEntityIndexBatchImpl.java | 34 ++++-
.../index/impl/EsEntityIndexImpl.java | 149 +++++++++++--------
.../persistence/index/impl/IndexingUtils.java | 2 +-
8 files changed, 137 insertions(+), 74 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/22beca2c/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
index ccb6221..b2a1a2a 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
@@ -219,7 +219,7 @@ public class IndexServiceImpl implements IndexService {
//collect results into a single batch
.collect( () -> ei.createBatch(), ( batch, candidateResult ) -> {
logger.debug( "Deindexing on edge {} for entity {} added to batch",searchEdge , entityId );
- batch.deindex( searchEdge, candidateResult );
+ batch.deindex( candidateResult );
} )
//return the future from the batch execution
.flatMap( batch ->Observable.just(batch.build()) );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/22beca2c/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/CandidateResult.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/CandidateResult.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/CandidateResult.java
index 2375509..9b1898c 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/CandidateResult.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/CandidateResult.java
@@ -32,10 +32,12 @@ import org.apache.usergrid.persistence.model.entity.Id;
public class CandidateResult implements EntityVersion {
private final Id entityId;
private final UUID entityVersion;
+ private final String docId;
- public CandidateResult( Id entityId, UUID entityVersion ) {
+ public CandidateResult( Id entityId, UUID entityVersion, String docId ) {
this.entityId = entityId;
this.entityVersion = entityVersion;
+ this.docId = docId;
}
@Override
@@ -48,6 +50,10 @@ public class CandidateResult implements EntityVersion {
return entityId;
}
+ public String getDocId() {
+ return docId;
+ }
+
@Override
public boolean equals( final Object o ) {
@@ -66,6 +72,9 @@ public class CandidateResult implements EntityVersion {
if ( !entityVersion.equals( that.entityVersion ) ) {
return false;
}
+ if ( !docId.equals( that.docId ) ) {
+ return false;
+ }
return true;
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/22beca2c/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexBatch.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexBatch.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexBatch.java
index 98652c1..17dd4d3 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexBatch.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexBatch.java
@@ -63,7 +63,7 @@ public interface EntityIndexBatch {
EntityIndexBatch deindex( final SearchEdge searchEdge, final Id id, final UUID version );
-
+ EntityIndexBatch deindex( final CandidateResult candidateResult);
/**
* get the batches
http://git-wip-us.apache.org/repos/asf/usergrid/blob/22beca2c/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
index e093d7d..33f199a 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
@@ -60,6 +60,8 @@ public interface IndexFig extends GuicyFig {
String ELASTICSEARCH_CLIENT_TYPE = "elasticsearch.client.type";
+ String ELASTICSEARCH_VERSION_QUERY_LIMIT = "elasticsearch.version_query_limit";
+
/**
* Comma-separated list of Elasticsearch hosts.
@@ -199,4 +201,8 @@ public interface IndexFig extends GuicyFig {
@Default("1000")
@Key( "elasticsearch_queue_error_sleep_ms" )
long getSleepTimeForQueueError();
+
+ @Default("1000")
+ @Key( ELASTICSEARCH_VERSION_QUERY_LIMIT )
+ int getVersionQueryLimit();
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/22beca2c/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/DeIndexOperation.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/DeIndexOperation.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/DeIndexOperation.java
index 4060dac..dbecf8a 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/DeIndexOperation.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/DeIndexOperation.java
@@ -61,6 +61,11 @@ public class DeIndexOperation implements BatchOperation {
this.documentId = createIndexDocId( applicationScope, id, version, searchEdge );
}
+ public DeIndexOperation( String[] indexes, String docId) {
+ this.indexes = indexes;
+ this.documentId = docId;
+ }
+
@Override
public void doOperation( final Client client, final BulkRequestBuilder bulkRequest ) {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/22beca2c/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
index 68830ca..a8fb751 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
@@ -84,23 +84,42 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
@Override
public EntityIndexBatch deindex( final SearchEdge searchEdge, final Id id, final UUID version ) {
- IndexValidationUtils.validateSearchEdge( searchEdge );
- ValidationUtils.verifyIdentity( id );
+ IndexValidationUtils.validateSearchEdge(searchEdge);
+ ValidationUtils.verifyIdentity(id);
ValidationUtils.verifyVersion( version );
String[] indexes = entityIndex.getIndexes();
//get the default index if no alias exists yet
if ( indexes == null || indexes.length == 0 ) {
- throw new IllegalStateException("No indexes exist for " + indexLocationStrategy.getAlias().getWriteAlias());
+ throw new IllegalStateException("No indexes exist for " + indexLocationStrategy.getAlias().getWriteAlias());
}
if ( log.isDebugEnabled() ) {
log.debug( "Deindexing to indexes {} with scope {} on edge {} with id {} and version {} ",
- new Object[] { indexes, applicationScope, searchEdge, id, version } );
+ new Object[] { indexes, applicationScope, searchEdge, id, version } );
}
- container.addDeIndexRequest( new DeIndexOperation( indexes, applicationScope, searchEdge, id, version ) );
+ container.addDeIndexRequest(new DeIndexOperation(indexes, applicationScope, searchEdge, id, version));
+
+ return this;
+ }
+
+ public EntityIndexBatch deindexWithDocId( final String docId ) {
+
+ String[] indexes = entityIndex.getIndexes();
+ //get the default index if no alias exists yet
+ if ( indexes == null || indexes.length == 0 ) {
+ throw new IllegalStateException("No indexes exist for " + indexLocationStrategy.getAlias().getWriteAlias());
+ }
+
+ if ( log.isDebugEnabled() ) {
+ log.debug( "Deindexing to indexes {} with with documentId {} ",
+ new Object[] { indexes, docId } );
+ }
+
+
+ container.addDeIndexRequest( new DeIndexOperation( indexes, docId ) );
return this;
}
@@ -117,6 +136,11 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
return deindex( searchEdge, entity.getId(), entity.getVersion() );
}
+ @Override
+ public EntityIndexBatch deindex( final CandidateResult entity ) {
+
+ return deindexWithDocId(entity.getDocId());
+ }
@Override
http://git-wip-us.apache.org/repos/asf/usergrid/blob/22beca2c/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 9be591e..0b7f2d7 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -65,6 +65,7 @@ import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.aggregations.metrics.sum.Sum;
import org.elasticsearch.search.aggregations.metrics.sum.SumBuilder;
+import org.elasticsearch.search.sort.SortOrder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -389,7 +390,7 @@ public class EsEntityIndexImpl implements EntityIndex,VersionedData {
final ParsedQuery parsedQuery = ParsedQueryBuilder.build(query);
final SearchRequestBuilder srb = searchRequest.getBuilder( searchEdge, searchTypes, parsedQuery, limit, offset )
- .setTimeout( TimeValue.timeValueMillis( queryTimeout ) );
+ .setTimeout(TimeValue.timeValueMillis(queryTimeout));
if ( logger.isDebugEnabled() ) {
logger.debug( "Searching index (read alias): {}\n nodeId: {}, edgeType: {}, \n type: {}\n query: {} ",
@@ -427,57 +428,62 @@ public class EsEntityIndexImpl implements EntityIndex,VersionedData {
*/
IndexValidationUtils.validateSearchEdge(edge);
- Preconditions.checkNotNull( entityId, "entityId cannot be null" );
+ Preconditions.checkNotNull(entityId, "entityId cannot be null");
SearchResponse searchResponse;
-
List<CandidateResult> candidates = new ArrayList<>();
- final ParsedQuery parsedQuery = ParsedQueryBuilder.build( "select *" );
-
- final SearchRequestBuilder srb = searchRequestBuilderStrategyV2.getBuilder();
-
- //I can't just search on the entity Id.
+ //never let the limit be less than 2 as there are potential indefinite paging issues
+ final int searchLimit = Math.max(2, indexFig.getVersionQueryLimit());
- FilterBuilder entityEdgeFilter = FilterBuilders.termFilter(IndexingUtils.EDGE_NODE_ID_FIELDNAME,
- IndexingUtils.nodeId(edge.getNodeId()));
+ final QueryBuilder entityQuery = QueryBuilders
+ .termQuery(IndexingUtils.EDGE_NODE_ID_FIELDNAME, IndexingUtils.nodeId(edge.getNodeId()));
- srb.setPostFilter(entityEdgeFilter);
+ final SearchRequestBuilder srb = searchRequestBuilderStrategyV2.getBuilder();
if ( logger.isDebugEnabled() ) {
- logger.debug( "Searching for marked versions in index (read alias): {}\n nodeId: {},\n query: {} ",
+ logger.debug( "Searching for edges in (read alias): {}\n nodeId: {},\n query: {} ",
this.alias.getReadAlias(),entityId, srb );
}
try {
- //Added For Graphite Metrics
-
- //set the timeout on the scroll cursor to 6 seconds and set the number of values returned per shard to 100.
- //The settings for the scroll aren't tested and so we aren't sure what vlaues would be best in a production enviroment
- //TODO: review this and make them not magic numbers when acking this PR.
- searchResponse = srb.setScroll( new TimeValue( 6000 ) ).setSize( 100 ).execute().actionGet();
+ long queryTimestamp = 0L;
while(true){
- //add search result hits to some sort of running tally of hits.
- candidates = aggregateScrollResults( candidates, searchResponse, null );
- SearchScrollRequestBuilder ssrb = searchRequestBuilderStrategyV2
- .getScrollBuilder( searchResponse.getScrollId() )
- .setScroll( new TimeValue( 6000 ) );
+ QueryBuilder timestampQuery = QueryBuilders
+ .rangeQuery(IndexingUtils.EDGE_TIMESTAMP_FIELDNAME)
+ .gte(queryTimestamp);
- //TODO: figure out how to log exactly what we're putting into elasticsearch
- // if ( logger.isDebugEnabled() ) {
- // logger.debug( "Scroll search using query: {} ",
- // ssrb.toString() );
- // }
+ QueryBuilder finalQuery = QueryBuilders
+ .boolQuery()
+ .must(entityQuery)
+ .must(timestampQuery);
- searchResponse = ssrb.execute().actionGet();
+ searchResponse = srb
+ .setQuery(finalQuery)
+ .setSize(searchLimit)
+ .addSort(IndexingUtils.EDGE_TIMESTAMP_FIELDNAME, SortOrder.ASC)
+ .execute()
+ .actionGet();
- if (searchResponse.getHits().getHits().length == 0) {
+ int responseSize = searchResponse.getHits().getHits().length;
+ if(responseSize == 0){
break;
}
+ // update queryTimestamp to be the timestamp of the last entity returned from the query
+ queryTimestamp = (long) searchResponse
+ .getHits().getAt(responseSize - 1)
+ .getSource().get(IndexingUtils.EDGE_TIMESTAMP_FIELDNAME);
+
+ candidates = aggregateScrollResults(candidates, searchResponse, null);
+
+ if(responseSize < searchLimit){
+
+ break;
+ }
}
}
@@ -488,7 +494,7 @@ public class EsEntityIndexImpl implements EntityIndex,VersionedData {
}
failureMonitor.success();
- return new CandidateResults( candidates, parsedQuery.getSelectFieldMappings());
+ return new CandidateResults( candidates, Collections.EMPTY_SET);
}
@@ -496,61 +502,74 @@ public class EsEntityIndexImpl implements EntityIndex,VersionedData {
public CandidateResults getAllEntityVersionsBeforeMarkedVersion( final Id entityId, final UUID markedVersion ) {
Preconditions.checkNotNull( entityId, "entityId cannot be null" );
- Preconditions.checkNotNull( markedVersion, "markedVersion cannot be null" );
+ Preconditions.checkNotNull(markedVersion, "markedVersion cannot be null");
ValidationUtils.verifyVersion(markedVersion);
SearchResponse searchResponse;
-
List<CandidateResult> candidates = new ArrayList<>();
- final ParsedQuery parsedQuery = ParsedQueryBuilder.build( "select *" );
+ final long markedTimestamp = markedVersion.timestamp();
- final SearchRequestBuilder srb = searchRequestBuilderStrategyV2.getBuilder();
-
- FilterBuilder entityIdFilter = FilterBuilders.termFilter(IndexingUtils.ENTITY_ID_FIELDNAME,
- IndexingUtils.entityId(entityId));
+ // never let the limit be less than 2 as there are potential indefinite paging issues
+ final int searchLimit = Math.max(2, indexFig.getVersionQueryLimit());
- srb.setPostFilter(entityIdFilter);
+ // this query will find the document for the entity itself
+ final QueryBuilder entityQuery = QueryBuilders
+ .termQuery(IndexingUtils.ENTITY_ID_FIELDNAME, IndexingUtils.entityId(entityId));
+ // this query will find all the documents where this entity is a source/target node
+ final QueryBuilder nodeQuery = QueryBuilders
+ .termQuery(IndexingUtils.EDGE_NODE_ID_FIELDNAME, IndexingUtils.nodeId(entityId));
-
- if ( logger.isDebugEnabled() ) {
- logger.debug( "Searching for marked versions in index (read alias): {}\n nodeId: {},\n query: {} ",
- this.alias.getReadAlias(),entityId, srb );
- }
+ final SearchRequestBuilder srb = searchRequestBuilderStrategyV2.getBuilder();
try {
- //Added For Graphite Metrics
- //set the timeout on the scroll cursor to 6 seconds and set the number of values returned per shard to 100.
- //The settings for the scroll aren't tested and so we aren't sure what vlaues would be best in a production enviroment
- //TODO: review this and make them not magic numbers when acking this PR.
- searchResponse = srb.setScroll( new TimeValue( 6000 ) ).setSize( 100 ).execute().actionGet();
+ long queryTimestamp = 0L;
- //list that will hold all of the search hits
+ while(true){
+ QueryBuilder timestampQuery = QueryBuilders
+ .rangeQuery(IndexingUtils.EDGE_TIMESTAMP_FIELDNAME)
+ .gte(queryTimestamp)
+ .lte(markedTimestamp);
+
+ QueryBuilder entityQueryWithTimestamp = QueryBuilders
+ .boolQuery()
+ .must(entityQuery)
+ .must(timestampQuery);
+
+ QueryBuilder finalQuery = QueryBuilders
+ .boolQuery()
+ .should(entityQueryWithTimestamp)
+ .should(nodeQuery)
+ .minimumNumberShouldMatch(1);
+
+ searchResponse = srb
+ .setQuery(finalQuery)
+ .setSize(searchLimit)
+ .addSort(IndexingUtils.EDGE_TIMESTAMP_FIELDNAME, SortOrder.ASC)
+ .execute()
+ .actionGet();
- while(true){
- //add search result hits to some sort of running tally of hits.
- candidates = aggregateScrollResults( candidates, searchResponse, markedVersion);
+ int responseSize = searchResponse.getHits().getHits().length;
+ if(responseSize == 0){
+ break;
+ }
+
+ candidates = aggregateScrollResults(candidates, searchResponse, markedVersion);
- SearchScrollRequestBuilder ssrb = searchRequestBuilderStrategyV2
- .getScrollBuilder( searchResponse.getScrollId() )
- .setScroll( new TimeValue( 6000 ) );
+ // update queryTimestamp to be the timestamp of the last entity returned from the query
+ queryTimestamp = (long) searchResponse
+ .getHits().getAt(responseSize - 1)
+ .getSource().get(IndexingUtils.EDGE_TIMESTAMP_FIELDNAME);
- //TODO: figure out how to log exactly what we're putting into elasticsearch
-// if ( logger.isDebugEnabled() ) {
-// logger.debug( "Scroll search using query: {} ",
-// ssrb.toString() );
-// }
- searchResponse = ssrb.execute().actionGet();
+ if(responseSize < searchLimit){
- if (searchResponse.getHits().getHits().length == 0) {
break;
}
-
}
}
catch ( Throwable t ) {
@@ -560,7 +579,7 @@ public class EsEntityIndexImpl implements EntityIndex,VersionedData {
}
failureMonitor.success();
- return new CandidateResults( candidates, parsedQuery.getSelectFieldMappings());
+ return new CandidateResults( candidates, Collections.EMPTY_SET);
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/22beca2c/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
index 9e06fa6..18cb928 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
@@ -241,7 +241,7 @@ public class IndexingUtils {
Id entityId = new SimpleId( UUID.fromString( entityUUID ), entityType );
- return new CandidateResult( entityId, UUID.fromString( versionUUID ) );
+ return new CandidateResult( entityId, UUID.fromString( versionUUID ), documentId );
}
[43/50] [abbrv] usergrid git commit: Merge commit
'refs/pull/421/head' of github.com:apache/usergrid
Posted by sn...@apache.org.
Merge commit 'refs/pull/421/head' of github.com:apache/usergrid
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/568d35d5
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/568d35d5
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/568d35d5
Branch: refs/heads/asf-site
Commit: 568d35d57e8a664a9eaf4ce9edf313d49e7e963b
Parents: b031d4d 73dcaeb
Author: Michael Russo <mi...@gmail.com>
Authored: Mon Nov 2 09:16:45 2015 -0800
Committer: Michael Russo <mi...@gmail.com>
Committed: Mon Nov 2 09:16:45 2015 -0800
----------------------------------------------------------------------
stack/loadtests/README.md | 14 +-
stack/loadtests/runAuditDeleteEntities.sh | 65 ++++++
.../runAuditGetAllAppCollectionEntities.sh | 6 +-
.../loadtests/runAuditGetCollectionEntities.sh | 6 +-
.../runAuditVerifyCollectionEntities.sh | 2 +-
stack/loadtests/runLoadEntitiesUnlimited.sh | 1 +
stack/loadtests/runOrgAppSetup.sh | 17 +-
.../datagenerators/FeederGenerator.scala | 16 +-
.../usergrid/enums/ConfigProperties.scala | 8 +-
.../apache/usergrid/enums/ScenarioType.scala | 3 +-
.../apache/usergrid/helpers/Extractors.scala | 18 +-
.../usergrid/scenarios/AuditScenarios.scala | 203 +++++++++++++------
.../scenarios/EntityCollectionScenarios.scala | 102 +++++++---
.../org/apache/usergrid/settings/Settings.scala | 42 ++--
.../usergrid/simulations/AuditSimulation.scala | 15 +-
.../simulations/ConfigurableSimulation.scala | 2 +-
stack/loadtests/testConfig.sh | 4 +-
17 files changed, 385 insertions(+), 139 deletions(-)
----------------------------------------------------------------------
[18/50] [abbrv] usergrid git commit: USERGRID-1044: print progress
during audit and log failures as such
Posted by sn...@apache.org.
USERGRID-1044: print progress during audit and log failures as such
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/2a09bf15
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/2a09bf15
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/2a09bf15
Branch: refs/heads/asf-site
Commit: 2a09bf15d8063199bf25b7e7f437929601f7193a
Parents: 2e70a72
Author: Mike Dunker <md...@apigee.com>
Authored: Wed Oct 28 07:28:51 2015 -0700
Committer: Mike Dunker <md...@apigee.com>
Committed: Wed Oct 28 07:28:51 2015 -0700
----------------------------------------------------------------------
.../org/apache/usergrid/datagenerators/FeederGenerator.scala | 8 ++++++++
.../scala/org/apache/usergrid/scenarios/AuditScenarios.scala | 6 +++---
.../usergrid/scenarios/EntityCollectionScenarios.scala | 8 +++++---
3 files changed, 16 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/2a09bf15/stack/loadtests/src/main/scala/org/apache/usergrid/datagenerators/FeederGenerator.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/datagenerators/FeederGenerator.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/datagenerators/FeederGenerator.scala
index 38db19a..a9225b2 100755
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/datagenerators/FeederGenerator.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/datagenerators/FeederGenerator.scala
@@ -255,6 +255,9 @@ object FeederGenerator {
val csvLines = if (Settings.feedAuditUuids) Source.fromFile(Settings.feedAuditUuidFilename).getLines().toArray else Array[String]()
val csvLinesLen = csvLines.length
var counter = new AtomicInteger(0)
+ var entityCounter: Long = 0L
+ var lastEntityCountPrinted: Long = 0L
+ var entityProgressCount: Long = Settings.entityProgressCount
override def hasNext: Boolean = true
@@ -284,6 +287,11 @@ object FeederGenerator {
val accessField = if (uuid != "") uuid else name
val queryField = if (uuid != "") s"uuid='$uuid'" else s"name='$name'"
+ if (entityProgressCount > 0L && entityCounter >= lastEntityCountPrinted + entityProgressCount) {
+ println(s"CSV Entity: $entityCounter")
+ lastEntityCountPrinted = entityCounter
+ }
+
Map("collectionName" -> collectionName, "name" -> name, "uuid" -> uuid, "modified" -> modified, "lastStatus" -> lastStatus,
"validEntity" -> validEntity, "accessField" -> accessField, "queryField" -> queryField)
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/2a09bf15/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala
index 1ce2bc2..d1e2c4b 100644
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala
@@ -202,15 +202,15 @@ object AuditScenarios {
Settings.incAuditNotFoundViaQuery()
println(s"NOT FOUND VIA QUERY: $collectionName.$reqName ($uuid)")
}
- session
+ session.markAsFailed
} else if (saveFailures) {
Settings.addAuditUuid(uuid, collectionName, reqName, modified, status, "Failure", lastStatus)
- session
+ session.markAsFailed
} else {
session.markAsFailed
}
- }).exitHereIfFailed
+ })
val getCollectionEntity = exec(
http("GET collection entity")
http://git-wip-us.apache.org/repos/asf/usergrid/blob/2a09bf15/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
index d5b3101..22b9f08 100644
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
@@ -241,16 +241,18 @@ object EntityCollectionScenarios {
val entityName = session("entityName").as[String]
val modified = if (status == 200) session(SessionVarModified).as[Long] else 0
val collectionName = session("collectionName").as[String]
+ Settings.addUuid(uuid, collectionName, entityName, modified, status)
if (status != 200) {
val bodyString = session(SessionVarBodyString).as[String]
println(s">>>>>>>> LOAD ERROR - Status: $status\nBody:\n$bodyString")
+ session.markAsFailed
+ } else {
+ session
}
- Settings.addUuid(uuid, collectionName, entityName, modified, status)
- session
} else {
session.markAsFailed
}
- }).exitHereIfFailed
+ })
}
)
[31/50] [abbrv] usergrid git commit: Rename new script to
multitenant-migrate and restore old script.
Posted by sn...@apache.org.
Rename new script to multitenant-migrate and restore old script.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/8af152e6
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/8af152e6
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/8af152e6
Branch: refs/heads/asf-site
Commit: 8af152e6bef74aa6259aff5a69428523da35d0bf
Parents: 57a613b
Author: Dave Johnson <sn...@apache.org>
Authored: Thu Oct 29 18:00:54 2015 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Thu Oct 29 18:00:54 2015 -0400
----------------------------------------------------------------------
stack/scripts/migrate_entity_data.py | 401 +++++------------
stack/scripts/multitenant_migrate.py | 703 ++++++++++++++++++++++++++++++
2 files changed, 816 insertions(+), 288 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/8af152e6/stack/scripts/migrate_entity_data.py
----------------------------------------------------------------------
diff --git a/stack/scripts/migrate_entity_data.py b/stack/scripts/migrate_entity_data.py
index 0edd319..9c01270 100644
--- a/stack/scripts/migrate_entity_data.py
+++ b/stack/scripts/migrate_entity_data.py
@@ -16,64 +16,30 @@
# under the License.
#
#
-# To migrate multiple tenants within one cluster.
#
-# STEP 1 - SETUP TENANT ONE TOMCAT RUNNING 2.1 NOT IN SERVICE AND INIT MIGRATION
+# Usage from a machine running Usergrid with the new Usergrid version:
#
-# Login to the Tomcat instance and run this command, specifying both superuser and tenant organization admin creds:
+# ######################################################
+# STEP 1 - BEFORE SWITCHING TRAFFIC TO NEW UG VERSION
+# ######################################################
#
-# python migrate_entity_data.py --org <org1name> --super <user>:<pass> --admin <user>:<pass> --init
+# python migrate_entity_data.py --user adminuser:adminpass
#
-# This command will setup and bootstrap the database, setup the migration system and update index mappings:
-# - /system/database/setup
-# - /system/database/bootstrap
-# - /system/migrate/run/migration-system
-# - /system/migrate/run/index_mapping_migration
+# The above command performs an appinfo migration and system re-index only. This creates indices in Elasticsearch with
+# the updated indexing strategy in the new Usergrid version.
#
-# Then it will migrate appinfos, re-index the management app and then for each of the specified org's apps
-# it will de-dup connections and re-index the app.
+# ######################################################
+# STEP 2 - AFTER SWITCHING TRAFFIC TO NEW UG VERSION
+# ######################################################
#
-# Write down the 'Re-index start' timestamp when this is finished.
+# python migrate_entity_data.py --user adminuser:adminpass --delta --date <timestamp>
#
-# STEP 2 - PUT TENANT ONE TOMCATS IN SERVICE AND DO DELTA MIGRATION
-#
-# On the same Tomcat instance and run this command with the --date timestamp you noted in the previous step:
-#
-# python migrate_entity_data.py --org <org1name> --super <user>:<pass> --admin <user>:<pass> --date <timestamp>
-#
-# Then it will migrate appinfos, re-index the management app and then for each of the specified org's apps
-# it will de-dup connections and re-index the app with a start-date specified so only data modified since
-# STEP 1 will be re-indexed.
-#
-# STEP 3 - SETUP TENANT TWO TOMCAT RUNNING 2.1 NOT IN SERVICE
-#
-# Login to the Tomcat instance and run this command, specifying both superuser and tenant organization admin creds:
-#
-# python migrate_entity_data.py --org <org2name> --super <user>:<pass> --admin <user>:<pass>
-#
-# This command will migrate appinfos, re-index the management app and then for each of the specified org's apps
-# it will de-dup connections and re-index the app.
-#
-# Write down the 'Re-index start' timestamp when this is finished.
+# The above command performs an appinfo migration, system re-index using a start date, and full data migration which
+# includes entity data. This step is necessary to ensure Usergrid starts reading and writing data from the latest
+# entity version, including delta indexing of any documents create during the time between STEP 1 and STEP 2. If
+# all data has already been migrated (running this a 2nd, 3rd, etc. time), then the appinfo migration will be skipped.
+
-# STEP 4 - PUT TENANT TWO TOMCATS IN SERVICE AND DO DELTA MIGRATION
-#
-# On the same Tomcat instance and run this command with the --date timestamp you noted in the previous step:
-#
-# python migrate_entity_data.py --org <org2name> --super <user>:<pass> --admin <user>:<pass> --date <timestamp>
-#
-# Then it will migrate appinfos, re-index the management app and then for each of the specified org's apps
-# it will de-dup connections and re-index the app with a start-date specified so only data modified since
-# STEP 1 will be re-indexed.
-#
-# STEP 5 - FULL DATA MIGRATION
-#
-# Login to any Tomcat instance in the cluster and run this command (admin user creds must be specificed but will be ignored):
-#
-# python migrate_entity_data.py --super <user>:<pass> --admin <user>:<pass> --full
-#
-# This command will run the full data migration.
-#
import sys
import logging
@@ -101,66 +67,40 @@ PLUGIN_ENTITYDATA = 'collections-entity-data'
PLUGIN_INDEX_MAPPING = 'index_mapping_migration'
PLUGIN_CORE_DATA = 'core-data'
-MANAGEMENT_APP_ID = 'b6768a08-b5d5-11e3-a495-11ddb1de66c8'
-
def parse_args():
parser = argparse.ArgumentParser(description='Usergrid Migration Tool')
+ parser.add_argument('--date',
+ help='A date from which to start the migration',
+ type=str)
+
parser.add_argument('--endpoint',
help='The endpoint to use for making API requests.',
type=str,
default='http://localhost:8080')
- parser.add_argument('--super',
- help='Superuser username and creds <user:pass>',
+ parser.add_argument('--user',
+ help='System Admin Credentials used to authenticate with Usergrid <user:pass>',
type=str,
required=True)
- parser.add_argument('--admin',
- help='Organization admin creds <user:pass>',
- type=str,
- required=True)
-
- parser.add_argument('--init',
- help='Init system and start first migration.',
- action='store_true',
- default=False)
-
- parser.add_argument('--org',
- help='Name of organization on which to run migration.',
- type=str,
- required=False)
-
- parser.add_argument('--date',
- help='A date from which to start the migration',
- type=str)
-
- parser.add_argument('--full',
- help='Run full data migration (last step in cluster migration).',
+ parser.add_argument('--delta',
+ help='Run a delta migration.',
action='store_true',
default=False)
my_args = parser.parse_args(sys.argv[1:])
arg_vars = vars(my_args)
-
- creds = arg_vars['super'].split(':')
- if len(creds) != 2:
- print('Superuser credentials not properly specified. Must be "-u <user:pass>". Exiting...')
- exit_on_error()
- else:
- arg_vars['superuser'] = creds[0]
- arg_vars['superpass'] = creds[1]
-
- creds = arg_vars['super'].split(':')
+ creds = arg_vars['user'].split(':')
if len(creds) != 2:
- print('Org admin credentials not properly specified. Must be "-u <user:pass>". Exiting...')
+ print('Credentials not properly specified. Must be "-u <user:pass>". Exiting...')
exit_on_error()
else:
- arg_vars['adminuser'] = creds[0]
- arg_vars['adminpass'] = creds[1]
+ arg_vars['user'] = creds[0]
+ arg_vars['pass'] = creds[1]
return arg_vars
@@ -178,13 +118,9 @@ class Migrate:
'full_data_migration_start': '',
'full_data_migration_end': ''}
self.logger = init_logging(self.__class__.__name__)
- self.super_user = self.args['superuser']
- self.super_pass = self.args['superpass']
- self.admin_user = self.args['adminuser']
- self.admin_pass = self.args['adminpass']
- self.org = self.args['org']
- self.init = self.args['init']
- self.full = self.args['full']
+ self.admin_user = self.args['user']
+ self.admin_pass = self.args['pass']
+ self.delta_migration = self.args['delta']
def run(self):
self.logger.info('Initializing...')
@@ -197,83 +133,63 @@ class Migrate:
try:
- if self.full:
+ self.run_database_setup()
- # Do full data migration and exit
+ # We need to check and roll the migration system to 1 if not already
+ migration_system_updated = self.is_migration_system_updated()
- self.start_fulldata_migration()
-
- self.metrics['full_data_migration_start'] = get_current_time()
- self.logger.info("Full Data Migration Started")
- is_migrated = False
- while not is_migrated:
+ if not migration_system_updated:
+ self.logger.info('Migration system needs to be updated. Updating migration system..')
+ self.start_migration_system_update()
+ while not migration_system_updated:
time.sleep(STATUS_INTERVAL_SECONDS)
- is_migrated = self.is_data_migrated()
- if is_migrated:
+ migration_system_updated = self.is_migration_system_updated()
+ if migration_system_updated:
break
- self.metrics['full_data_migration_end'] = get_current_time()
- self.logger.info("Full Data Migration completed")
+ index_mapping_updated = self.is_index_mapping_updated()
- self.log_metrics()
- self.logger.info("Finished...")
-
- return
-
-
- if self.init:
-
- # Init the migration system as this is the first migration done on the cluster
-
- self.run_database_setup()
-
- migration_system_updated = self.is_migration_system_updated()
-
- if not migration_system_updated:
- self.logger.info('Migration system needs to be updated. Updating migration system..')
- self.start_migration_system_update()
- while not migration_system_updated:
- time.sleep(STATUS_INTERVAL_SECONDS)
- migration_system_updated = self.is_migration_system_updated()
- if migration_system_updated:
- break
+ if not index_mapping_updated:
+ self.logger.info('Index Mapping needs to be updated. Updating index mapping..')
+ self.start_index_mapping_migration()
+ while not index_mapping_updated:
+ time.sleep(STATUS_INTERVAL_SECONDS)
+ index_mapping_updated = self.is_index_mapping_updated()
+ if index_mapping_updated:
+ break
- index_mapping_updated = self.is_index_mapping_updated()
+ # Run AppInfo migration only when both appinfos and collection entity data have not been migrated
+ if not self.is_data_migrated():
- if not index_mapping_updated:
- self.logger.info('Index Mapping needs to be updated. Updating index mapping..')
- self.start_index_mapping_migration()
- while not index_mapping_updated:
- time.sleep(STATUS_INTERVAL_SECONDS)
- index_mapping_updated = self.is_index_mapping_updated()
- if index_mapping_updated:
- break
+ #Migrate app info
+ if self.is_appinfo_migrated():
+ self.logger.info('AppInfo already migrated. Resetting version for re-migration.')
+ self.reset_appinfo_migration()
+ time.sleep(STATUS_INTERVAL_SECONDS)
+ self.start_appinfo_migration()
+ self.logger.info('AppInfo Migration Started.')
+ self.metrics['appinfo_migration_start'] = get_current_time()
- # Migrate app info
+ is_appinfo_migrated = False
+ while not is_appinfo_migrated:
+ is_appinfo_migrated = self.is_appinfo_migrated()
+ time.sleep(STATUS_INTERVAL_SECONDS)
+ if is_appinfo_migrated:
+ self.metrics['appinfo_migration_end'] = get_current_time()
+ break
+ self.logger.info('AppInfo Migration Ended.')
- if self.is_appinfo_migrated():
- self.logger.info('AppInfo already migrated. Resetting version for re-migration.')
- self.reset_appinfo_migration()
- time.sleep(STATUS_INTERVAL_SECONDS)
- self.start_appinfo_migration()
- self.logger.info('AppInfo Migration Started.')
- self.metrics['appinfo_migration_start'] = get_current_time()
+ else:
+ self.logger.info('Full Data Migration previously ran... skipping AppInfo migration.')
- is_appinfo_migrated = False
- while not is_appinfo_migrated:
- is_appinfo_migrated = self.is_appinfo_migrated()
- time.sleep(STATUS_INTERVAL_SECONDS)
- if is_appinfo_migrated:
- self.metrics['appinfo_migration_end'] = get_current_time()
- break
- self.logger.info('AppInfo Migration Ended.')
- # Reindex management app
+ # We need to check and roll index mapping version to 1 if not already there
- job = self.start_app_reindex(MANAGEMENT_APP_ID)
+ # Perform system re-index (it will grab date from input if provided)
+ job = self.start_reindex()
self.metrics['reindex_start'] = get_current_time()
self.logger.info('Started Re-index. Job=[%s]', job)
is_running = True
@@ -286,44 +202,33 @@ class Migrate:
self.logger.info("Finished Re-index. Job=[%s]", job)
self.metrics['reindex_end'] = get_current_time()
+ # Only when we do a delta migration do we run the full data migration (includes appinfo and entity data)
+ if self.delta_migration:
- # Dedup and re-index all of organization's apps
-
- app_ids = self.get_app_ids()
- for app_id in app_ids:
-
- # De-dep app
- # job = self.start_dedup(app_id)
- # self.metrics['dedup_start_' + app_id] = get_current_time()
- # self.logger.info('Started dedup. App=[%s], Job=[%s]', app_id, job)
- # is_running = True
- # while is_running:
- # time.sleep(STATUS_INTERVAL_SECONDS)
- # is_running = self.is_dedup_running(job)
- # if not is_running:
- # break
- #
- # self.logger.info("Finished dedup. App=[%s], Job=[%s]", app_id, job)
- # self.metrics['dedup_end_' + app_id] = get_current_time()
-
- # Re-index app
- job = self.start_app_reindex(app_id)
- self.metrics['reindex_start_' + app_id] = get_current_time()
- self.logger.info('Started Re-index. App=[%s], Job=[%s]', app_id, job)
- is_running = True
- while is_running:
+ self.logger.info('Delta option provided. Performing full data migration...')
+ if self.is_data_migrated():
+ self.reset_data_migration()
+ time.sleep(STATUS_INTERVAL_SECONDS)
+ self.is_data_migrated()
+
+ # self.start_core_data_migration()
+ self.start_fulldata_migration()
+
+ self.metrics['full_data_migration_start'] = get_current_time()
+ self.logger.info("Full Data Migration Started")
+ is_migrated = False
+ while not is_migrated:
time.sleep(STATUS_INTERVAL_SECONDS)
- is_running = self.is_reindex_running(job)
- if not is_running:
+ is_migrated = self.is_data_migrated()
+ if is_migrated:
break
- self.logger.info("Finished Re-index. App=[%s], Job=[%s]", app_id, job)
- self.metrics['reindex_end_' + app_id] = get_current_time()
+ self.metrics['full_data_migration_end'] = get_current_time()
+ self.logger.info("Full Data Migration completed")
self.log_metrics()
self.logger.info("Finished...")
-
except KeyboardInterrupt:
self.log_metrics()
self.logger.error('Keyboard interrupted migration. Please run again to ensure the migration finished.')
@@ -332,10 +237,6 @@ class Migrate:
url = self.endpoint + '/system/database/setup'
return url
- def get_database_bootstrap_url(self):
- url = self.endpoint + '/system/database/bootstrap'
- return url
-
def get_migration_url(self):
url = self.endpoint + '/system/migrate/run'
return url
@@ -348,30 +249,28 @@ class Migrate:
url = self.endpoint + '/system/migrate/status'
return url
- def get_dedup_url(self):
- url = self.endpoint + '/system/connection/dedup'
- return url
-
def get_reindex_url(self):
url = self.endpoint + '/system/index/rebuild'
return url
def get_management_reindex_url(self):
- url = self.get_reindex_url() + "/management"
- return url
+ url = self.get_reindex_url() + "/management"
+ return url
+
def start_core_data_migration(self):
- try:
- r = requests.put(url=self.get_migration_url(), auth=(self.admin_user, self.super_pass))
- response = r.json()
- return response
- except requests.exceptions.RequestException as e:
- self.logger.error('Failed to start migration, %s', e)
- exit_on_error(str(e))
+ try:
+ r = requests.put(url=self.get_migration_url(), auth=(self.admin_user, self.admin_pass))
+ response = r.json()
+ return response
+ except requests.exceptions.RequestException as e:
+ self.logger.error('Failed to start migration, %s', e)
+ exit_on_error(str(e))
+
def start_fulldata_migration(self):
try:
- r = requests.put(url=self.get_migration_url(), auth=(self.admin_user, self.super_pass))
+ r = requests.put(url=self.get_migration_url(), auth=(self.admin_user, self.admin_pass))
response = r.json()
return response
except requests.exceptions.RequestException as e:
@@ -380,9 +279,9 @@ class Migrate:
def start_migration_system_update(self):
try:
- # TODO fix this URL
+ #TODO fix this URL
migrateUrl = self.get_migration_url() + '/' + PLUGIN_MIGRATION_SYSTEM
- r = requests.put(url=migrateUrl, auth=(self.admin_user, self.super_pass))
+ r = requests.put(url=migrateUrl, auth=(self.admin_user, self.admin_pass))
response = r.json()
return response
except requests.exceptions.RequestException as e:
@@ -392,7 +291,7 @@ class Migrate:
def run_database_setup(self):
try:
setupUrl = self.get_database_setup_url()
- r = requests.put(url=setupUrl, auth=(self.super_user, self.super_pass))
+ r = requests.put(url=setupUrl, auth=(self.admin_user, self.admin_pass))
if r.status_code != 200:
exit_on_error('Database Setup Failed')
@@ -400,21 +299,10 @@ class Migrate:
self.logger.error('Failed to run database setup, %s', e)
exit_on_error(str(e))
- def run_database_bootstrap(self):
- try:
- setupUrl = self.get_database_bootstrap_url()
- r = requests.put(url=setupUrl, auth=(self.super_user, self.super_pass))
- if r.status_code != 200:
- exit_on_error('Database Bootstrap Failed')
-
- except requests.exceptions.RequestException as e:
- self.logger.error('Failed to run database bootstrap, %s', e)
- exit_on_error(str(e))
-
def start_index_mapping_migration(self):
try:
migrateUrl = self.get_migration_url() + '/' + PLUGIN_INDEX_MAPPING
- r = requests.put(url=migrateUrl, auth=(self.super_user, self.super_pass))
+ r = requests.put(url=migrateUrl, auth=(self.admin_user, self.admin_pass))
response = r.json()
return response
except requests.exceptions.RequestException as e:
@@ -435,7 +323,7 @@ class Migrate:
version = TARGET_ENTITY_DATA_VERSION - 1
body = json.dumps({PLUGIN_ENTITYDATA: version, PLUGIN_APPINFO: version})
try:
- r = requests.put(url=self.get_reset_migration_url(), data=body, auth=(self.super_user, self.super_pass))
+ r = requests.put(url=self.get_reset_migration_url(), data=body, auth=(self.admin_user, self.admin_pass))
response = r.json()
self.logger.info('Resetting data migration versions to %s=[%s] '
'and %s=[%s]', PLUGIN_ENTITYDATA, version, PLUGIN_APPINFO, version)
@@ -448,7 +336,7 @@ class Migrate:
version = TARGET_APPINFO_VERSION - 1
body = json.dumps({PLUGIN_APPINFO: version})
try:
- r = requests.put(url=self.get_reset_migration_url(), data=body, auth=(self.super_user, self.super_pass))
+ r = requests.put(url=self.get_reset_migration_url(), data=body, auth=(self.admin_user, self.admin_pass))
response = r.json()
self.logger.info('Resetting appinfo migration versions to %s=[%s]', PLUGIN_APPINFO, version)
return response
@@ -527,7 +415,7 @@ class Migrate:
def check_data_migration_status(self):
try:
- r = requests.get(url=self.get_migration_status_url(), auth=(self.super_user, self.super_pass))
+ r = requests.get(url=self.get_migration_status_url(), auth=(self.admin_user, self.admin_pass))
if r.status_code == 200:
response = r.json()
return response
@@ -542,20 +430,20 @@ class Migrate:
status_url = self.get_reindex_url()+'/' + job
try:
- r = requests.get(url=status_url, auth=(self.super_user, self.super_pass))
+ r = requests.get(url=status_url, auth=(self.admin_user, self.admin_pass))
response = r.json()
return response['status']
except requests.exceptions.RequestException as e:
self.logger.error('Failed to get reindex status, %s', e)
# exit_on_error()
- def start_app_reindex(self, appId):
+ def start_reindex(self):
body = ""
if self.start_date is not None:
body = json.dumps({'updated': self.start_date})
try:
- r = requests.post(url=self.get_reindex_url(), data=body, auth=(self.super_user, self.super_pass))
+ r = requests.post(url=self.get_reindex_url(), data=body, auth=(self.admin_user, self.admin_pass))
if r.status_code == 200:
response = r.json()
@@ -575,41 +463,6 @@ class Migrate:
else:
return False
- def get_dedup_status(self, job):
- status_url = self.get_dedup_url()+'/' + job
- try:
- r = requests.get(url=status_url, auth=(self.super_user, self.super_pass))
- print r.text
- response = r.json()
- return response['status']
- except requests.exceptions.RequestException as e:
- self.logger.error('Failed to get dedup status, %s', e)
- # exit_on_error()
-
- def start_dedup(self, app_id):
- body = ""
- try:
- r = requests.post(url=self.get_dedup_url() + "/" + app_id, data=body, auth=(self.super_user, self.super_pass))
- if r.status_code == 200:
- response = r.json()
- print r.text
- return response['status']['jobStatusId']
- else:
- self.logger.error('Failed to start dedup, %s', r)
- exit_on_error(str(r))
-
- except requests.exceptions.RequestException as e:
- self.logger.error('Unable to make API request for dedup, %s', e)
- exit_on_error(str(e))
-
- def is_dedup_running(self, job):
- status = self.get_dedup_status(job)
- self.logger.info('Dedup status=[%s]', status)
- if status != "COMPLETE":
- return True
- else:
- return False
-
def is_endpoint_available(self):
try:
@@ -637,34 +490,6 @@ class Migrate:
)
- def get_app_ids(self):
-
- try:
-
- url = self.endpoint + "/management/token"
- body = json.dumps({"grant_type":"password","username":self.admin_user,"password":self.admin_pass})
- r = requests.post(url=url, data=body)
- if ( r.status_code != 200 ):
- print "Error logging in: " + r.text
- return
-
- access_token = r.json()["access_token"]
-
- url = self.endpoint + "/management/orgs/" + self.org + "/apps?access_token=" + access_token
- r = requests.get(url=url)
- if r.status_code != 200:
- exit_on_error('Cannot get app ids: ' + r.text)
-
- apps = r.json()["data"]
- app_ids = []
- for appId in apps.values():
- app_ids.append(appId)
-
- return app_ids;
-
- except requests.exceptions.RequestException as e:
- self.logger.error('Unable to get list of application ids, %s', e)
- exit_on_error(str(e))
def get_current_time():
return str(int(time.time()*1000))
http://git-wip-us.apache.org/repos/asf/usergrid/blob/8af152e6/stack/scripts/multitenant_migrate.py
----------------------------------------------------------------------
diff --git a/stack/scripts/multitenant_migrate.py b/stack/scripts/multitenant_migrate.py
new file mode 100644
index 0000000..62e46af
--- /dev/null
+++ b/stack/scripts/multitenant_migrate.py
@@ -0,0 +1,703 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#
+# To migrate multiple tenants within one cluster.
+#
+# STEP 1 - SETUP TENANT ONE TOMCAT RUNNING 2.1 NOT IN SERVICE AND INIT MIGRATION
+#
+# Login to the Tomcat instance and run this command, specifying both superuser and tenant organization admin creds:
+#
+# python migrate_entity_data.py --org <org1name> --super <user>:<pass> --admin <user>:<pass> --init
+#
+# This command will setup and bootstrap the database, setup the migration system and update index mappings:
+# - /system/database/setup
+# - /system/database/bootstrap
+# - /system/migrate/run/migration-system
+# - /system/migrate/run/index_mapping_migration
+#
+# Then it will migrate appinfos, re-index the management app and then for each of the specified org's apps
+# it will de-dup connections and re-index the app.
+#
+# Write down the 'Re-index start' timestamp when this is finished.
+#
+# STEP 2 - PUT TENANT ONE TOMCATS IN SERVICE AND DO DELTA MIGRATION
+#
+# On the same Tomcat instance and run this command with the --date timestamp you noted in the previous step:
+#
+# python migrate_entity_data.py --org <org1name> --super <user>:<pass> --admin <user>:<pass> --date <timestamp>
+#
+# Then it will migrate appinfos, re-index the management app and then for each of the specified org's apps
+# it will de-dup connections and re-index the app with a start-date specified so only data modified since
+# STEP 1 will be re-indexed.
+#
+# STEP 3 - SETUP TENANT TWO TOMCAT RUNNING 2.1 NOT IN SERVICE
+#
+# Login to the Tomcat instance and run this command, specifying both superuser and tenant organization admin creds:
+#
+# python migrate_entity_data.py --org <org2name> --super <user>:<pass> --admin <user>:<pass>
+#
+# This command will migrate appinfos, re-index the management app and then for each of the specified org's apps
+# it will de-dup connections and re-index the app.
+#
+# Write down the 'Re-index start' timestamp when this is finished.
+
+# STEP 4 - PUT TENANT TWO TOMCATS IN SERVICE AND DO DELTA MIGRATION
+#
+# On the same Tomcat instance and run this command with the --date timestamp you noted in the previous step:
+#
+# python migrate_entity_data.py --org <org2name> --super <user>:<pass> --admin <user>:<pass> --date <timestamp>
+#
+# Then it will migrate appinfos, re-index the management app and then for each of the specified org's apps
+# it will de-dup connections and re-index the app with a start-date specified so only data modified since
+# STEP 1 will be re-indexed.
+#
+# STEP 5 - FULL DATA MIGRATION
+#
+# Login to any Tomcat instance in the cluster and run this command (admin user creds must be specificed but will be ignored):
+#
+# python migrate_entity_data.py --super <user>:<pass> --admin <user>:<pass> --full
+#
+# This command will run the full data migration.
+#
+
+import sys
+import logging
+from logging.handlers import RotatingFileHandler
+import argparse
+import time
+import requests
+import json
+
+
+# Version expected in status response post-migration for entity and app-info data
+TARGET_APPINFO_VERSION=2
+TARGET_ENTITY_DATA_VERSION=2
+TARGET_CORE_DATA_VERSION=2
+TARGET_MIGRATION_SYSTEM_VERSION = 1
+TARGET_INDEX_MAPPING_VERSION = 2
+
+# Set an interval (in seconds) for checking if re-index and/or migration has finished
+STATUS_INTERVAL_SECONDS = 2
+
+# Set plugin names
+PLUGIN_MIGRATION_SYSTEM = 'migration-system'
+PLUGIN_APPINFO = 'appinfo-migration'
+PLUGIN_ENTITYDATA = 'collections-entity-data'
+PLUGIN_INDEX_MAPPING = 'index_mapping_migration'
+PLUGIN_CORE_DATA = 'core-data'
+
+MANAGEMENT_APP_ID = 'b6768a08-b5d5-11e3-a495-11ddb1de66c8'
+
+
+
+def parse_args():
+ parser = argparse.ArgumentParser(description='Usergrid Migration Tool')
+
+ parser.add_argument('--endpoint',
+ help='The endpoint to use for making API requests.',
+ type=str,
+ default='http://localhost:8080')
+
+ parser.add_argument('--super',
+ help='Superuser username and creds <user:pass>',
+ type=str,
+ required=True)
+
+ parser.add_argument('--admin',
+ help='Organization admin creds <user:pass>',
+ type=str,
+ required=True)
+
+ parser.add_argument('--init',
+ help='Init system and start first migration.',
+ action='store_true',
+ default=False)
+
+ parser.add_argument('--org',
+ help='Name of organization on which to run migration.',
+ type=str,
+ required=False)
+
+ parser.add_argument('--date',
+ help='A date from which to start the migration',
+ type=str)
+
+ parser.add_argument('--full',
+ help='Run full data migration (last step in cluster migration).',
+ action='store_true',
+ default=False)
+
+ my_args = parser.parse_args(sys.argv[1:])
+
+ arg_vars = vars(my_args)
+
+ creds = arg_vars['super'].split(':')
+ if len(creds) != 2:
+ print('Superuser credentials not properly specified. Must be "-u <user:pass>". Exiting...')
+ exit_on_error()
+ else:
+ arg_vars['superuser'] = creds[0]
+ arg_vars['superpass'] = creds[1]
+
+ creds = arg_vars['super'].split(':')
+ if len(creds) != 2:
+ print('Org admin credentials not properly specified. Must be "-u <user:pass>". Exiting...')
+ exit_on_error()
+ else:
+ arg_vars['adminuser'] = creds[0]
+ arg_vars['adminpass'] = creds[1]
+
+ return arg_vars
+
+
+
+class Migrate:
+ def __init__(self):
+ self.args = parse_args()
+ self.start_date = self.args['date']
+ self.endpoint = self.args['endpoint']
+ self.metrics = {'reindex_start': '',
+ 'reindex_end': '',
+ 'appinfo_migration_start': '',
+ 'appinfo_migration_end': '',
+ 'full_data_migration_start': '',
+ 'full_data_migration_end': ''}
+ self.logger = init_logging(self.__class__.__name__)
+ self.super_user = self.args['superuser']
+ self.super_pass = self.args['superpass']
+ self.admin_user = self.args['adminuser']
+ self.admin_pass = self.args['adminpass']
+ self.org = self.args['org']
+ self.init = self.args['init']
+ self.full = self.args['full']
+
+ def run(self):
+ self.logger.info('Initializing...')
+
+ if not self.is_endpoint_available():
+ exit_on_error('Endpoint is not available, aborting')
+
+ if self.start_date is not None:
+ self.logger.info("Date Provided. Re-index will run from date=[%s]", self.start_date)
+
+ try:
+
+ if self.full:
+
+ # Do full data migration and exit
+
+ self.start_fulldata_migration()
+
+ self.metrics['full_data_migration_start'] = get_current_time()
+ self.logger.info("Full Data Migration Started")
+ is_migrated = False
+ while not is_migrated:
+ time.sleep(STATUS_INTERVAL_SECONDS)
+ is_migrated = self.is_data_migrated()
+ if is_migrated:
+ break
+
+ self.metrics['full_data_migration_end'] = get_current_time()
+ self.logger.info("Full Data Migration completed")
+
+ self.log_metrics()
+ self.logger.info("Finished...")
+
+ return
+
+
+ if self.init:
+
+ # Init the migration system as this is the first migration done on the cluster
+
+ self.run_database_setup()
+
+ migration_system_updated = self.is_migration_system_updated()
+
+ if not migration_system_updated:
+ self.logger.info('Migration system needs to be updated. Updating migration system..')
+ self.start_migration_system_update()
+ while not migration_system_updated:
+ time.sleep(STATUS_INTERVAL_SECONDS)
+ migration_system_updated = self.is_migration_system_updated()
+ if migration_system_updated:
+ break
+
+ index_mapping_updated = self.is_index_mapping_updated()
+
+ if not index_mapping_updated:
+ self.logger.info('Index Mapping needs to be updated. Updating index mapping..')
+ self.start_index_mapping_migration()
+ while not index_mapping_updated:
+ time.sleep(STATUS_INTERVAL_SECONDS)
+ index_mapping_updated = self.is_index_mapping_updated()
+ if index_mapping_updated:
+ break
+
+
+ # Migrate app info
+
+ if self.is_appinfo_migrated():
+ self.logger.info('AppInfo already migrated. Resetting version for re-migration.')
+ self.reset_appinfo_migration()
+ time.sleep(STATUS_INTERVAL_SECONDS)
+
+ self.start_appinfo_migration()
+ self.logger.info('AppInfo Migration Started.')
+ self.metrics['appinfo_migration_start'] = get_current_time()
+
+ is_appinfo_migrated = False
+ while not is_appinfo_migrated:
+ is_appinfo_migrated = self.is_appinfo_migrated()
+ time.sleep(STATUS_INTERVAL_SECONDS)
+ if is_appinfo_migrated:
+ self.metrics['appinfo_migration_end'] = get_current_time()
+ break
+ self.logger.info('AppInfo Migration Ended.')
+
+
+ # Reindex management app
+
+ job = self.start_app_reindex(MANAGEMENT_APP_ID)
+ self.metrics['reindex_start'] = get_current_time()
+ self.logger.info('Started Re-index. Job=[%s]', job)
+ is_running = True
+ while is_running:
+ time.sleep(STATUS_INTERVAL_SECONDS)
+ is_running = self.is_reindex_running(job)
+ if not is_running:
+ break
+
+ self.logger.info("Finished Re-index. Job=[%s]", job)
+ self.metrics['reindex_end'] = get_current_time()
+
+
+ # Dedup and re-index all of organization's apps
+
+ app_ids = self.get_app_ids()
+ for app_id in app_ids:
+
+ # De-dup app
+ job = self.start_dedup(app_id)
+ self.logger.info('Started dedup. App=[%s], Job=[%s]', app_id, job)
+ is_running = True
+ while is_running:
+ time.sleep(STATUS_INTERVAL_SECONDS)
+ is_running = self.is_dedup_running(job)
+ if not is_running:
+ break
+
+ self.logger.info("Finished dedup. App=[%s], Job=[%s]", app_id, job)
+ self.metrics['dedup_end_' + app_id] = get_current_time()
+
+ # Re-index app
+ job = self.start_app_reindex(app_id)
+ self.metrics['reindex_start_' + app_id] = get_current_time()
+ self.logger.info('Started Re-index. App=[%s], Job=[%s]', app_id, job)
+ is_running = True
+ while is_running:
+ time.sleep(STATUS_INTERVAL_SECONDS)
+ is_running = self.is_reindex_running(job)
+ if not is_running:
+ break
+
+ self.logger.info("Finished Re-index. App=[%s], Job=[%s]", app_id, job)
+ self.metrics['reindex_end_' + app_id] = get_current_time()
+
+ self.log_metrics()
+ self.logger.info("Finished...")
+
+
+ except KeyboardInterrupt:
+ self.log_metrics()
+ self.logger.error('Keyboard interrupted migration. Please run again to ensure the migration finished.')
+
+ def get_database_setup_url(self):
+ url = self.endpoint + '/system/database/setup'
+ return url
+
+ def get_database_bootstrap_url(self):
+ url = self.endpoint + '/system/database/bootstrap'
+ return url
+
+ def get_migration_url(self):
+ url = self.endpoint + '/system/migrate/run'
+ return url
+
+ def get_reset_migration_url(self):
+ url = self.endpoint + '/system/migrate/set'
+ return url
+
+ def get_migration_status_url(self):
+ url = self.endpoint + '/system/migrate/status'
+ return url
+
+ def get_dedup_url(self):
+ url = self.endpoint + '/system/connection/dedup'
+ return url
+
+ def get_reindex_url(self):
+ url = self.endpoint + '/system/index/rebuild'
+ return url
+
+ def get_management_reindex_url(self):
+ url = self.get_reindex_url() + "/management"
+ return url
+
+ def start_core_data_migration(self):
+ try:
+ r = requests.put(url=self.get_migration_url(), auth=(self.admin_user, self.super_pass))
+ response = r.json()
+ return response
+ except requests.exceptions.RequestException as e:
+ self.logger.error('Failed to start migration, %s', e)
+ exit_on_error(str(e))
+
+ def start_fulldata_migration(self):
+ try:
+ r = requests.put(url=self.get_migration_url(), auth=(self.admin_user, self.super_pass))
+ response = r.json()
+ return response
+ except requests.exceptions.RequestException as e:
+ self.logger.error('Failed to start migration, %s', e)
+ exit_on_error(str(e))
+
+ def start_migration_system_update(self):
+ try:
+ # TODO fix this URL
+ migrateUrl = self.get_migration_url() + '/' + PLUGIN_MIGRATION_SYSTEM
+ r = requests.put(url=migrateUrl, auth=(self.admin_user, self.super_pass))
+ response = r.json()
+ return response
+ except requests.exceptions.RequestException as e:
+ self.logger.error('Failed to start migration, %s', e)
+ exit_on_error(str(e))
+
+ def run_database_setup(self):
+ try:
+ setupUrl = self.get_database_setup_url()
+ r = requests.put(url=setupUrl, auth=(self.super_user, self.super_pass))
+ if r.status_code != 200:
+ exit_on_error('Database Setup Failed')
+
+ except requests.exceptions.RequestException as e:
+ self.logger.error('Failed to run database setup, %s', e)
+ exit_on_error(str(e))
+
+ def run_database_bootstrap(self):
+ try:
+ setupUrl = self.get_database_bootstrap_url()
+ r = requests.put(url=setupUrl, auth=(self.super_user, self.super_pass))
+ if r.status_code != 200:
+ exit_on_error('Database Bootstrap Failed')
+
+ except requests.exceptions.RequestException as e:
+ self.logger.error('Failed to run database bootstrap, %s', e)
+ exit_on_error(str(e))
+
+ def start_index_mapping_migration(self):
+ try:
+ migrateUrl = self.get_migration_url() + '/' + PLUGIN_INDEX_MAPPING
+ r = requests.put(url=migrateUrl, auth=(self.super_user, self.super_pass))
+ response = r.json()
+ return response
+ except requests.exceptions.RequestException as e:
+ self.logger.error('Failed to start migration, %s', e)
+ exit_on_error(str(e))
+
+ def start_appinfo_migration(self):
+ try:
+ migrateUrl = self.get_migration_url() + '/' + PLUGIN_APPINFO
+ r = requests.put(url=migrateUrl, auth=(self.admin_user, self.admin_pass))
+ response = r.json()
+ return response
+ except requests.exceptions.RequestException as e:
+ self.logger.error('Failed to start migration, %s', e)
+ exit_on_error(str(e))
+
+ def reset_data_migration(self):
+ version = TARGET_ENTITY_DATA_VERSION - 1
+ body = json.dumps({PLUGIN_ENTITYDATA: version, PLUGIN_APPINFO: version})
+ try:
+ r = requests.put(url=self.get_reset_migration_url(), data=body, auth=(self.super_user, self.super_pass))
+ response = r.json()
+ self.logger.info('Resetting data migration versions to %s=[%s] '
+ 'and %s=[%s]', PLUGIN_ENTITYDATA, version, PLUGIN_APPINFO, version)
+ return response
+ except requests.exceptions.RequestException as e:
+ self.logger.error('Failed to reset full data migration versions, %s', e)
+ exit_on_error(str(e))
+
+ def reset_appinfo_migration(self):
+ version = TARGET_APPINFO_VERSION - 1
+ body = json.dumps({PLUGIN_APPINFO: version})
+ try:
+ r = requests.put(url=self.get_reset_migration_url(), data=body, auth=(self.super_user, self.super_pass))
+ response = r.json()
+ self.logger.info('Resetting appinfo migration versions to %s=[%s]', PLUGIN_APPINFO, version)
+ return response
+ except requests.exceptions.RequestException as e:
+ self.logger.error('Failed to reset appinfo migration version, %s', e)
+ exit_on_error(str(e))
+
+ def is_data_migrated(self):
+ status = self.check_data_migration_status()
+ if status is not None:
+ entity_version = status['data'][PLUGIN_ENTITYDATA]
+ appinfo_version = status['data'][PLUGIN_APPINFO]
+ core_data_version = status['data'][PLUGIN_CORE_DATA]
+
+ if entity_version == TARGET_ENTITY_DATA_VERSION and appinfo_version == TARGET_APPINFO_VERSION and core_data_version == TARGET_CORE_DATA_VERSION:
+ self.logger.info('Full Data Migration status=[COMPLETE], %s=[%s], '
+ '%s=[%s], %s=%s',
+ PLUGIN_ENTITYDATA,
+ entity_version,
+ PLUGIN_APPINFO,
+ appinfo_version,
+ PLUGIN_CORE_DATA,
+ core_data_version)
+ return True
+ else:
+ self.logger.info('Full Data Migration status=[NOTSTARTED/INPROGRESS]')
+ return False
+
+ def is_appinfo_migrated(self):
+ status = self.check_data_migration_status()
+ if status is not None:
+ appinfo_version = status['data'][PLUGIN_APPINFO]
+
+ if appinfo_version == TARGET_APPINFO_VERSION:
+ self.logger.info('AppInfo Migration status=[COMPLETE],'
+ '%s=[%s]',
+ PLUGIN_APPINFO,
+ appinfo_version)
+ return True
+ else:
+ self.logger.info('AppInfo Migration status=[NOTSTARTED/INPROGRESS]')
+ return False
+
+ def is_migration_system_updated(self):
+ status = self.check_data_migration_status()
+ if status is not None:
+ migration_system_version = status['data'][PLUGIN_MIGRATION_SYSTEM]
+
+ if migration_system_version == TARGET_MIGRATION_SYSTEM_VERSION:
+ self.logger.info('Migration System CURRENT, %s=[%s]',
+ PLUGIN_MIGRATION_SYSTEM,
+ migration_system_version)
+ return True
+ else:
+ self.logger.info('Migration System OLD, %s=[%s]',
+ PLUGIN_MIGRATION_SYSTEM,
+ migration_system_version)
+ return False
+
+ def is_index_mapping_updated(self):
+ status = self.check_data_migration_status()
+ if status is not None:
+ index_mapping_version = status['data'][PLUGIN_INDEX_MAPPING]
+
+ if index_mapping_version == TARGET_INDEX_MAPPING_VERSION:
+ self.logger.info('Index Mapping CURRENT, %s=[%s]',
+ PLUGIN_INDEX_MAPPING,
+ index_mapping_version)
+ return True
+ else:
+ self.logger.info('Index Mapping OLD, %s=[%s]',
+ PLUGIN_INDEX_MAPPING,
+ index_mapping_version)
+ return False
+
+ def check_data_migration_status(self):
+
+ try:
+ r = requests.get(url=self.get_migration_status_url(), auth=(self.super_user, self.super_pass))
+ if r.status_code == 200:
+ response = r.json()
+ return response
+ else:
+ self.logger.error('Failed to check migration status, %s', r)
+ return
+ except requests.exceptions.RequestException as e:
+ self.logger.error('Failed to check migration status, %s', e)
+ # exit_on_error()
+
+ def get_reindex_status(self, job):
+ status_url = self.get_reindex_url()+'/' + job
+
+ try:
+ r = requests.get(url=status_url, auth=(self.super_user, self.super_pass))
+ response = r.json()
+ return response['status']
+ except requests.exceptions.RequestException as e:
+ self.logger.error('Failed to get reindex status, %s', e)
+ # exit_on_error()
+
+ def start_app_reindex(self, appId):
+ body = ""
+ if self.start_date is not None:
+ body = json.dumps({'updated': self.start_date})
+
+ try:
+ r = requests.post(url=self.get_reindex_url(), data=body, auth=(self.super_user, self.super_pass))
+
+ if r.status_code == 200:
+ response = r.json()
+ return response['jobId']
+ else:
+ self.logger.error('Failed to start reindex, %s', r)
+ exit_on_error(str(r))
+ except requests.exceptions.RequestException as e:
+ self.logger.error('Unable to make API request for reindex, %s', e)
+ exit_on_error(str(e))
+
+ def is_reindex_running(self, job):
+ status = self.get_reindex_status(job)
+ self.logger.info('Re-index status=[%s]', status)
+ if status != "COMPLETE":
+ return True
+ else:
+ return False
+
+ def get_dedup_status(self, job):
+ status_url = self.get_dedup_url()+'/' + job
+ try:
+ r = requests.get(url=status_url, auth=(self.super_user, self.super_pass))
+ response = r.json()
+ return response['status']['status']
+ except requests.exceptions.RequestException as e:
+ self.logger.error('Failed to get dedup status, %s', e)
+ # exit_on_error()
+
+ def start_dedup(self, app_id):
+ body = ""
+ try:
+ r = requests.post(url=self.get_dedup_url() + "/" + app_id, data=body, auth=(self.super_user, self.super_pass))
+ if r.status_code == 200:
+ response = r.json()
+ return response['status']['jobStatusId']
+ else:
+ self.logger.error('Failed to start dedup, %s', r)
+ exit_on_error(str(r))
+
+ except requests.exceptions.RequestException as e:
+ self.logger.error('Unable to make API request for dedup, %s', e)
+ exit_on_error(str(e))
+
+ def is_dedup_running(self, job):
+ status = self.get_dedup_status(job)
+ self.logger.info('Dedup status=[%s]', status)
+ if status != "COMPLETE":
+ return True
+ else:
+ return False
+
+ def is_endpoint_available(self):
+
+ try:
+ r = requests.get(url=self.endpoint+'/status')
+ if r.status_code == 200:
+ return True
+ except requests.exceptions.RequestException as e:
+ self.logger.error('Endpoint is unavailable, %s', str(e))
+ return False
+
+ def log_metrics(self):
+ self.logger.info(
+ 'Re-index start=[%s], ' +
+ 'Re-index end =[%s], ' +
+ 'Full Data Migration start=[%s], ' +
+ 'Full Data Migration end=[%s] ' +
+ 'AppInfo Migration start=[%s], ' +
+ 'AppInfo Migration end=[%s] ',
+ self.metrics['reindex_start'],
+ self.metrics['reindex_end'],
+ self.metrics['full_data_migration_start'],
+ self.metrics['full_data_migration_end'],
+ self.metrics['appinfo_migration_start'],
+ self.metrics['appinfo_migration_end']
+
+ )
+
+ def get_app_ids(self):
+
+ try:
+
+ url = self.endpoint + "/management/token"
+ body = json.dumps({"grant_type":"password","username":self.admin_user,"password":self.admin_pass})
+ r = requests.post(url=url, data=body)
+ if ( r.status_code != 200 ):
+ print "Error logging in: " + r.text
+ return
+
+ access_token = r.json()["access_token"]
+
+ url = self.endpoint + "/management/orgs/" + self.org + "/apps?access_token=" + access_token
+ r = requests.get(url=url)
+ if r.status_code != 200:
+ exit_on_error('Cannot get app ids: ' + r.text)
+
+ apps = r.json()["data"]
+ app_ids = []
+ for appId in apps.values():
+ app_ids.append(appId)
+
+ print app_ids
+
+ return app_ids
+
+ except requests.exceptions.RequestException as e:
+ self.logger.error('Unable to get list of application ids, %s', e)
+ exit_on_error(str(e))
+
+def get_current_time():
+ return str(int(time.time()*1000))
+
+
+def exit_on_error(e=""):
+ print ('Exiting migration script due to error: ' + str(e))
+ sys.exit(1)
+
+
+def init_logging(name):
+
+ logger = logging.getLogger(name)
+ log_file_name = './migration.log'
+ log_formatter = logging.Formatter(fmt='%(asctime)s [%(name)s] %(levelname)s %(message)s',
+ datefmt='%Y-%m-%d %H:%M:%S')
+
+ rotating_file = logging.handlers.RotatingFileHandler(filename=log_file_name,
+ mode='a',
+ maxBytes=104857600,
+ backupCount=10)
+ rotating_file.setFormatter(log_formatter)
+ rotating_file.setLevel(logging.INFO)
+ logger.addHandler(rotating_file)
+ logger.setLevel(logging.INFO)
+
+ stdout_logger = logging.StreamHandler(sys.stdout)
+ stdout_logger.setFormatter(log_formatter)
+ stdout_logger.setLevel(logging.INFO)
+ logger.addHandler(stdout_logger)
+
+ return logger
+
+if __name__ == '__main__':
+
+ migration = Migrate()
+ migration.run()
[49/50] [abbrv] usergrid git commit: Added 'usergrid.cluster_name
property=usergrid' to Usergrid 2 installation doc.
Posted by sn...@apache.org.
Added 'usergrid.cluster_name property=usergrid' to Usergrid 2 installation doc.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/c3299bc2
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/c3299bc2
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/c3299bc2
Branch: refs/heads/asf-site
Commit: c3299bc2178bf31c7ce912e2a2daf6f97fb4a911
Parents: a7b6a9c
Author: Dave Johnson <sn...@apache.org>
Authored: Fri Nov 6 15:37:12 2015 -0500
Committer: Dave Johnson <sn...@apache.org>
Committed: Fri Nov 6 15:37:12 2015 -0500
----------------------------------------------------------------------
content/docs/_sources/installation/ug2-deploy-to-tomcat.txt | 2 ++
content/docs/installation/ug2-deploy-to-tomcat.html | 2 ++
docs/installation/ug2-deploy-to-tomcat.md | 2 ++
3 files changed, 6 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c3299bc2/content/docs/_sources/installation/ug2-deploy-to-tomcat.txt
----------------------------------------------------------------------
diff --git a/content/docs/_sources/installation/ug2-deploy-to-tomcat.txt b/content/docs/_sources/installation/ug2-deploy-to-tomcat.txt
index 049b909..752b4f3 100644
--- a/content/docs/_sources/installation/ug2-deploy-to-tomcat.txt
+++ b/content/docs/_sources/installation/ug2-deploy-to-tomcat.txt
@@ -76,6 +76,8 @@ Example __usergrid-deployment.properties__ file
```
# core persistence properties
+usergrid.cluster_name property=usergrid
+
cassandra.embedded=false
cassandra.version=1.2.18
cassandra.timeout=2000
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c3299bc2/content/docs/installation/ug2-deploy-to-tomcat.html
----------------------------------------------------------------------
diff --git a/content/docs/installation/ug2-deploy-to-tomcat.html b/content/docs/installation/ug2-deploy-to-tomcat.html
index 4831130..d6c039a 100644
--- a/content/docs/installation/ug2-deploy-to-tomcat.html
+++ b/content/docs/installation/ug2-deploy-to-tomcat.html
@@ -352,6 +352,8 @@ it however you like!</p>
<h2>Example <strong>usergrid-deployment.properties</strong> file<a class="headerlink" href="#example-usergrid-deployment-properties-file" title="Permalink to this headline">¶</a></h2>
<div class="highlight-python"><div class="highlight"><pre># core persistence properties
+usergrid.cluster_name property=usergrid
+
cassandra.embedded=false
cassandra.version=1.2.18
cassandra.timeout=2000
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c3299bc2/docs/installation/ug2-deploy-to-tomcat.md
----------------------------------------------------------------------
diff --git a/docs/installation/ug2-deploy-to-tomcat.md b/docs/installation/ug2-deploy-to-tomcat.md
index 049b909..752b4f3 100644
--- a/docs/installation/ug2-deploy-to-tomcat.md
+++ b/docs/installation/ug2-deploy-to-tomcat.md
@@ -76,6 +76,8 @@ Example __usergrid-deployment.properties__ file
```
# core persistence properties
+usergrid.cluster_name property=usergrid
+
cassandra.embedded=false
cassandra.version=1.2.18
cassandra.timeout=2000
[22/50] [abbrv] usergrid git commit: First and untested stab at
multi-tenant migration.
Posted by sn...@apache.org.
First and untested stab at multi-tenant migration.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/d79bf4c3
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/d79bf4c3
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/d79bf4c3
Branch: refs/heads/asf-site
Commit: d79bf4c36810fdab578a86e4de0ae725f1aa6e75
Parents: 2686054
Author: Dave Johnson <sn...@apache.org>
Authored: Thu Oct 29 08:56:14 2015 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Thu Oct 29 08:56:14 2015 -0400
----------------------------------------------------------------------
stack/scripts/create_test_data.py | 213 +++++++++++++++++++++
stack/scripts/migrate_entity_data.py | 301 +++++++++++++++++++++---------
2 files changed, 429 insertions(+), 85 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/d79bf4c3/stack/scripts/create_test_data.py
----------------------------------------------------------------------
diff --git a/stack/scripts/create_test_data.py b/stack/scripts/create_test_data.py
new file mode 100644
index 0000000..de85da0
--- /dev/null
+++ b/stack/scripts/create_test_data.py
@@ -0,0 +1,213 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import sys
+import logging
+from logging.handlers import RotatingFileHandler
+import argparse
+import time
+import requests
+import json
+
+# Creates two organizations each with two apps each with three collections each with 100 entities
+# Caller must provide a "slug" string which will be used as a prefix for all names
+#
+# For example, if the slug is mytest then:
+# Orgs will be named mytest_org0 and mytest_org1
+# Apps will be named mytest_org0_app0 and so on
+# Collections will be named mytest_org0_app0_col0 and so on
+# Entities will be named mytest_org0_app0_col0_entity and so on
+# Org admins will be named mytest_org0_admin and mytest_org1_admin (both with password test)
+
+def parse_args():
+ parser = argparse.ArgumentParser(description="Usergrid Test Data Creation Tool")
+
+ parser.add_argument("--endpoint",
+ help="The endpoint to use for making API requests.",
+ type=str,
+ default="http://localhost:8080")
+
+ parser.add_argument("--user",
+ help="System Admin Credentials used to authenticate with Usergrid <user:pass>",
+ type=str,
+ required=True)
+
+ parser.add_argument("--slug",
+ help="Unique string to be used to name organization, applications and other things create",
+ type=str,
+ required=True)
+
+ my_args = parser.parse_args(sys.argv[1:])
+
+ arg_vars = vars(my_args)
+ creds = arg_vars["user"].split(":")
+ if len(creds) != 2:
+ print("Credentials not properly specified. Must be '-u <user:pass>'. Exiting...")
+ exit_on_error()
+ else:
+ arg_vars["user"] = creds[0]
+ arg_vars["pass"] = creds[1]
+
+ return arg_vars
+
+
+class Creator:
+ def __init__(self):
+ self.args = parse_args()
+ self.endpoint = self.args["endpoint"]
+ self.logger = init_logging(self.__class__.__name__)
+ self.admin_user = self.args["user"]
+ self.admin_pass = self.args["pass"]
+ self.slug = self.args["slug"]
+
+ def run(self):
+ self.logger.info("Initializing...")
+
+ if not self.is_endpoint_available():
+ exit_on_error("Endpoint is not available, aborting")
+
+ for orgIndex in range(2):
+ orgName = self.slug + "_org" + str(orgIndex)
+ orgUser = orgName + "_admin"
+ orgEmail = orgUser + "@example.com"
+
+ url = self.endpoint + "/management/orgs"
+ body = json.dumps({"username":orgUser, "email":orgEmail, "password":"test", "organization":orgName })
+ r = requests.post(url=url, data=body, auth=(self.admin_user, self.admin_pass))
+ if ( r.status_code >= 400 ):
+ print "Error creating organization " + orgName + ": " + r.text
+ return
+
+ print "Created org " + orgName
+
+ url = self.endpoint + "/management/token"
+ body = json.dumps({"grant_type":"password","username":orgUser,"password":"test"})
+ r = requests.post(url=url, data=body)
+ if ( r.status_code != 200 ):
+ print "Error logging into organization " + orgName + ": " + r.text
+ return
+
+ accessToken = r.json()["access_token"]
+
+ for appIndex in range(2):
+ appName = orgName + "_app" + str(appIndex)
+
+ url = self.endpoint + "/management/orgs/" + orgName + "/apps?access_token=" + accessToken
+ body = json.dumps({"name":appName})
+ r = requests.post(url=url, data=body, auth=(self.admin_user, self.admin_pass))
+ if ( r.status_code >= 400 ):
+ print "Error creating application" + appName + ": " + r.text
+ return
+
+ print " Created app: " + orgName + "/" + appName
+ appUrl = self.endpoint + "/" + orgName + "/" + appName
+ time.sleep(2)
+
+ for userIndex in range(2):
+ userName = appName + "_user" + str(userIndex)
+ email = userName + "@example.com"
+
+ url = appUrl + "/users?access_token=" + accessToken
+ body = json.dumps({"name":userName, "username":userName, "email":email, "password":"test"})
+ r = requests.post(url=url, data=body)
+ if ( r.status_code >= 400 ):
+ print "Error creating user " + userName + ": " + r.text
+ return
+
+ for colIndex in range(3):
+ colName = appName + "_col" + str(colIndex)
+ print " Creating collection: " + colName
+
+ for entityIndex in range(100):
+ entityName = colName + "_entity" + str(entityIndex)
+
+ url = appUrl + "/" + colName + "s?access_token=" + accessToken
+ body = json.dumps({"name":entityName})
+ r = requests.post(url=url, data=body)
+ if ( r.status_code >= 400 ):
+ print "Error creating entity" + userName + ": " + r.text
+ retur
+
+ # connect entities in collection 0 to collection 1
+ for entityIndex in range(100):
+ sourceCollection = appName + "_col0s"
+ sourceName = appName + "_col0_entity" + str(entityIndex)
+ targetName = appName + "_col1_entity" + str(entityIndex)
+ targetType = appName + "_col1"
+ url = appUrl + "/" + sourceCollection + "/" + sourceName + "/has/" + targetType + "/" + targetName
+ r = requests.post(url=url + "?access_token=" + accessToken)
+ if ( r.status_code >= 400 ):
+ print "Error connecting entity " + sourceName + " to " + targetName + ": " + r.text
+ print "url is: " + url
+ return
+
+ # connect entities in collection 1 to collection 2
+ for entityIndex in range(100):
+ sourceCollection = appName + "_col1s"
+ sourceName = appName + "_col1_entity" + str(entityIndex)
+ targetName = appName + "_col2_entity" + str(entityIndex)
+ targetType = appName + "_col2"
+ url = appUrl + "/" + sourceCollection + "/" + sourceName + "/has/" + targetType + "/" + targetName
+ r = requests.post(url=url + "?access_token=" + accessToken)
+ if ( r.status_code >= 400 ):
+ print "Error connecting entity " + sourceName + " to " + targetName + ": " + r.text
+ print "url is: " + url
+ return
+
+ def is_endpoint_available(self):
+
+ try:
+ r = requests.get(url=self.endpoint+"/status")
+ if r.status_code == 200:
+ return True
+ except requests.exceptions.RequestException as e:
+ self.logger.error("Endpoint is unavailable, %s", str(e))
+ return False
+
+
+def exit_on_error(e=""):
+ print ("Exiting script due to error: " + str(e))
+ sys.exit(1)
+
+
+def init_logging(name):
+
+ logger = logging.getLogger(name)
+ log_file_name = "./create-test-data.log"
+ log_formatter = logging.Formatter(fmt="%(asctime)s [%(name)s] %(levelname)s %(message)s",
+ datefmt="%Y-%m-%d %H:%M:%S")
+
+ rotating_file = logging.handlers.RotatingFileHandler(filename=log_file_name,
+ mode="a",
+ maxBytes=104857600,
+ backupCount=10)
+ rotating_file.setFormatter(log_formatter)
+ rotating_file.setLevel(logging.INFO)
+ logger.addHandler(rotating_file)
+ logger.setLevel(logging.INFO)
+
+ stdout_logger = logging.StreamHandler(sys.stdout)
+ stdout_logger.setFormatter(log_formatter)
+ stdout_logger.setLevel(logging.INFO)
+ logger.addHandler(stdout_logger)
+
+ return logger
+
+if __name__ == "__main__":
+
+ creator = Creator()
+ creator.run()
http://git-wip-us.apache.org/repos/asf/usergrid/blob/d79bf4c3/stack/scripts/migrate_entity_data.py
----------------------------------------------------------------------
diff --git a/stack/scripts/migrate_entity_data.py b/stack/scripts/migrate_entity_data.py
index 36f73c0..f576108 100644
--- a/stack/scripts/migrate_entity_data.py
+++ b/stack/scripts/migrate_entity_data.py
@@ -16,30 +16,50 @@
# under the License.
#
#
+# To migrate multiple tenants within one cluster.
#
-# Usage from a machine running Usergrid with the new Usergrid version:
+# STEP 1 - SETUP TENANT ONE TOMCAT RUNNING 2.1 NOT IN SERVICE AND INIT MIGRATION
#
-# ######################################################
-# STEP 1 - BEFORE SWITCHING TRAFFIC TO NEW UG VERSION
-# ######################################################
+# python migrate_entity_data.py --org <org1name> --user <superuser>:<superpass> --init
#
-# python migrate_entity_data.py --user adminuser:adminpass
+# This command will setup and bootstrap the database, setup the migration system and update index mappings:
+# - /system/database/setup
+# - /system/database/bootstrap
+# - /system/migrate/run/migration-system
+# - /system/migrate/run/index_mapping_migration
#
-# The above command performs an appinfo migration and system re-index only. This creates indices in Elasticsearch with
-# the updated indexing strategy in the new Usergrid version.
+# Then it will migrate appinfos, re-index the management app and then for each of the specified org's apps
+# it will de-dup connections and re-index the app.
#
-# ######################################################
-# STEP 2 - AFTER SWITCHING TRAFFIC TO NEW UG VERSION
-# ######################################################
+# STEP 2 - PUT TENANT ONE TOMCATS IN SERVICE AND DO DELTA MIGRATION
#
-# python migrate_entity_data.py --user adminuser:adminpass --delta --date <timestamp>
+# python migrate_entity_data.py --org <org1name> --user <superuser>:<superpass> --date
+#
+# Then it will migrate appinfos, re-index the management app and then for each of the specified org's apps
+# it will de-dup connections and re-index the app with a start-date specified so only data modified since
+# STEP 1 will be re-indexed.
+#
+# STEP 3 - SETUP TENENT TWO TOMCAT RUNNING 2.1 NOT IN SERVICE
+#
+# python migrate_entity_data.py --org <org2name> --user <superuser>:<superpass> --date
+#
+# This command will migrate appinfos, re-index the management app and then for each of the specified org's apps
+# it will de-dup connections and re-index the app.
+#
+# STEP 4 - PUT TENANT TWO TOMCATS IN SERVICE AND DO DELTA MIGRATION
+#
+# python migrate_entity_data.py --org <org2name> --user <superuser>:<superpass> --date
+#
+# Then it will migrate appinfos, re-index the management app and then for each of the specified org's apps
+# it will de-dup connections and re-index the app with a start-date specified so only data modified since
+# STEP 1 will be re-indexed.
+#
+# STEP 5 - FULL DATA MIGRATION
+#
+# python migrate_entity_data.py --user <superuser>:<superpass> --full
+#
+# This command will run the full data migration.
#
-# The above command performs an appinfo migration, system re-index using a start date, and full data migration which
-# includes entity data. This step is necessary to ensure Usergrid starts reading and writing data from the latest
-# entity version, including delta indexing of any documents create during the time between STEP 1 and STEP 2. If
-# all data has already been migrated (running this a 2nd, 3rd, etc. time), then the appinfo migration will be skipped.
-
-
import sys
import logging
@@ -72,10 +92,6 @@ PLUGIN_CORE_DATA = 'core-data'
def parse_args():
parser = argparse.ArgumentParser(description='Usergrid Migration Tool')
- parser.add_argument('--date',
- help='A date from which to start the migration',
- type=str)
-
parser.add_argument('--endpoint',
help='The endpoint to use for making API requests.',
type=str,
@@ -86,8 +102,22 @@ def parse_args():
type=str,
required=True)
- parser.add_argument('--delta',
- help='Run a delta migration.',
+ parser.add_argument('--init',
+ help='Init system and start first migration.',
+ action='store_true',
+ default=False)
+
+ parser.add_argument('--org',
+ help='Name of organization on which to run migration.',
+ type=str,
+ required=False)
+
+ parser.add_argument('--date',
+ help='A date from which to start the migration',
+ type=str)
+
+ parser.add_argument('--full',
+ help='Run full data migration (last step in cluster migration).',
action='store_true',
default=False)
@@ -120,7 +150,9 @@ class Migrate:
self.logger = init_logging(self.__class__.__name__)
self.admin_user = self.args['user']
self.admin_pass = self.args['pass']
- self.delta_migration = self.args['delta']
+ self.org = self.args['org']
+ self.init = self.args['init']
+ self.full = self.args['full']
def run(self):
self.logger.info('Initializing...')
@@ -133,62 +165,82 @@ class Migrate:
try:
- self.run_database_setup()
+ if self.full:
- # We need to check and roll the migration system to 1 if not already
- migration_system_updated = self.is_migration_system_updated()
+ # Do full data migration and exit
- if not migration_system_updated:
- self.logger.info('Migration system needs to be updated. Updating migration system..')
- self.start_migration_system_update()
- while not migration_system_updated:
+ self.start_fulldata_migration()
+
+ self.metrics['full_data_migration_start'] = get_current_time()
+ self.logger.info("Full Data Migration Started")
+ is_migrated = False
+ while not is_migrated:
time.sleep(STATUS_INTERVAL_SECONDS)
- migration_system_updated = self.is_migration_system_updated()
- if migration_system_updated:
+ is_migrated = self.is_data_migrated()
+ if is_migrated:
break
- index_mapping_updated = self.is_index_mapping_updated()
+ self.metrics['full_data_migration_end'] = get_current_time()
+ self.logger.info("Full Data Migration completed")
- if not index_mapping_updated:
- self.logger.info('Index Mapping needs to be updated. Updating index mapping..')
- self.start_index_mapping_migration()
- while not index_mapping_updated:
- time.sleep(STATUS_INTERVAL_SECONDS)
- index_mapping_updated = self.is_index_mapping_updated()
- if index_mapping_updated:
- break
+ self.log_metrics()
+ self.logger.info("Finished...")
- # Run AppInfo migration only when both appinfos and collection entity data have not been migrated
- if not self.is_data_migrated():
+ return
- #Migrate app info
- if self.is_appinfo_migrated():
- self.logger.info('AppInfo already migrated. Resetting version for re-migration.')
- self.reset_appinfo_migration()
- time.sleep(STATUS_INTERVAL_SECONDS)
- self.start_appinfo_migration()
- self.logger.info('AppInfo Migration Started.')
- self.metrics['appinfo_migration_start'] = get_current_time()
+ if self.init:
- is_appinfo_migrated = False
- while not is_appinfo_migrated:
- is_appinfo_migrated = self.is_appinfo_migrated()
- time.sleep(STATUS_INTERVAL_SECONDS)
- if is_appinfo_migrated:
- self.metrics['appinfo_migration_end'] = get_current_time()
- break
- self.logger.info('AppInfo Migration Ended.')
+ # Init the migration system as this is the first migration done on the cluster
+ self.run_database_setup()
- else:
- self.logger.info('Full Data Migration previously ran... skipping AppInfo migration.')
+ migration_system_updated = self.is_migration_system_updated()
+ if not migration_system_updated:
+ self.logger.info('Migration system needs to be updated. Updating migration system..')
+ self.start_migration_system_update()
+ while not migration_system_updated:
+ time.sleep(STATUS_INTERVAL_SECONDS)
+ migration_system_updated = self.is_migration_system_updated()
+ if migration_system_updated:
+ break
+ index_mapping_updated = self.is_index_mapping_updated()
- # We need to check and roll index mapping version to 1 if not already there
+ if not index_mapping_updated:
+ self.logger.info('Index Mapping needs to be updated. Updating index mapping..')
+ self.start_index_mapping_migration()
+ while not index_mapping_updated:
+ time.sleep(STATUS_INTERVAL_SECONDS)
+ index_mapping_updated = self.is_index_mapping_updated()
+ if index_mapping_updated:
+ break
+
+
+ # Migrate app info
+
+ if self.is_appinfo_migrated():
+ self.logger.info('AppInfo already migrated. Resetting version for re-migration.')
+ self.reset_appinfo_migration()
+ time.sleep(STATUS_INTERVAL_SECONDS)
+
+ self.start_appinfo_migration()
+ self.logger.info('AppInfo Migration Started.')
+ self.metrics['appinfo_migration_start'] = get_current_time()
+
+ is_appinfo_migrated = False
+ while not is_appinfo_migrated:
+ is_appinfo_migrated = self.is_appinfo_migrated()
+ time.sleep(STATUS_INTERVAL_SECONDS)
+ if is_appinfo_migrated:
+ self.metrics['appinfo_migration_end'] = get_current_time()
+ break
+ self.logger.info('AppInfo Migration Ended.')
+
+
+ # Reindex management app
- # Perform system re-index (it will grab date from input if provided)
job = self.start_reindex()
self.metrics['reindex_start'] = get_current_time()
self.logger.info('Started Re-index. Job=[%s]', job)
@@ -202,33 +254,44 @@ class Migrate:
self.logger.info("Finished Re-index. Job=[%s]", job)
self.metrics['reindex_end'] = get_current_time()
- # Only when we do a delta migration do we run the full data migration (includes appinfo and entity data)
- if self.delta_migration:
- self.logger.info('Delta option provided. Performing full data migration...')
- if self.is_data_migrated():
- self.reset_data_migration()
- time.sleep(STATUS_INTERVAL_SECONDS)
- self.is_data_migrated()
+ # Dedup and re-index all of organization's apps
- # self.start_core_data_migration()
- self.start_fulldata_migration()
+ app_ids = self.get_app_ids()
+ for app_id in app_ids:
- self.metrics['full_data_migration_start'] = get_current_time()
- self.logger.info("Full Data Migration Started")
- is_migrated = False
- while not is_migrated:
+ # De-dep app
+ job = self.start_app_dedup(app_id)
+ self.metrics['dedup_start_' + app_id] = get_current_time()
+ self.logger.info('Started dedup. App=[%s], Job=[%s]', app_id, job)
+ is_running = True
+ while is_running:
time.sleep(STATUS_INTERVAL_SECONDS)
- is_migrated = self.is_data_migrated()
- if is_migrated:
+ is_running = self.is_reindex_running(job)
+ if not is_running:
break
- self.metrics['full_data_migration_end'] = get_current_time()
- self.logger.info("Full Data Migration completed")
+ self.logger.info("Finished dedup. App=[%s], Job=[%s]", app_id, job)
+ self.metrics['dedup_end_' + app_id] = get_current_time()
+
+ # Re-index app
+ job = self.start_app_reindex(app_id)
+ self.metrics['reindex_start_' + app_id] = get_current_time()
+ self.logger.info('Started Re-index. App=[%s], Job=[%s]', app_id, job)
+ is_running = True
+ while is_running:
+ time.sleep(STATUS_INTERVAL_SECONDS)
+ is_running = self.is_reindex_running(job)
+ if not is_running:
+ break
+
+ self.logger.info("Finished Re-index. App=[%s], Job=[%s]", app_id, job)
+ self.metrics['reindex_end_' + app_id] = get_current_time()
self.log_metrics()
self.logger.info("Finished...")
+
except KeyboardInterrupt:
self.log_metrics()
self.logger.error('Keyboard interrupted migration. Please run again to ensure the migration finished.')
@@ -237,6 +300,10 @@ class Migrate:
url = self.endpoint + '/system/database/setup'
return url
+ def get_database_bootstrap_url(self):
+ url = self.endpoint + '/system/database/bootstrap'
+ return url
+
def get_migration_url(self):
url = self.endpoint + '/system/migrate/run'
return url
@@ -249,6 +316,10 @@ class Migrate:
url = self.endpoint + '/system/migrate/status'
return url
+ def get_dedup_url(self):
+ url = self.endpoint + '/system/connection/dedup'
+ return url
+
def get_reindex_url(self):
url = self.endpoint + '/system/index/rebuild'
return url
@@ -257,7 +328,6 @@ class Migrate:
url = self.get_reindex_url() + "/management"
return url
-
def start_core_data_migration(self):
try:
r = requests.put(url=self.get_migration_url(), auth=(self.admin_user, self.admin_pass))
@@ -267,7 +337,6 @@ class Migrate:
self.logger.error('Failed to start migration, %s', e)
exit_on_error(str(e))
-
def start_fulldata_migration(self):
try:
r = requests.put(url=self.get_migration_url(), auth=(self.admin_user, self.admin_pass))
@@ -279,7 +348,7 @@ class Migrate:
def start_migration_system_update(self):
try:
- #TODO fix this URL
+ # TODO fix this URL
migrateUrl = self.get_migration_url() + '/' + PLUGIN_MIGRATION_SYSTEM
r = requests.put(url=migrateUrl, auth=(self.admin_user, self.admin_pass))
response = r.json()
@@ -299,6 +368,17 @@ class Migrate:
self.logger.error('Failed to run database setup, %s', e)
exit_on_error(str(e))
+ def run_database_bootstrap(self):
+ try:
+ setupUrl = self.get_database_bootstrap_url()
+ r = requests.put(url=setupUrl, auth=(self.admin_user, self.admin_pass))
+ if r.status_code != 200:
+ exit_on_error('Database Bootstrap Failed')
+
+ except requests.exceptions.RequestException as e:
+ self.logger.error('Failed to run database bootstrap, %s', e)
+ exit_on_error(str(e))
+
def start_index_mapping_migration(self):
try:
migrateUrl = self.get_migration_url() + '/' + PLUGIN_INDEX_MAPPING
@@ -437,7 +517,7 @@ class Migrate:
self.logger.error('Failed to get reindex status, %s', e)
# exit_on_error()
- def start_reindex(self):
+ def start_app_reindex(self, appId):
body = ""
if self.start_date is not None:
body = json.dumps({'updated': self.start_date})
@@ -463,6 +543,39 @@ class Migrate:
else:
return False
+ def get_dedup_status(self, job):
+ status_url = self.get_dedup_url()+'/' + job
+ try:
+ r = requests.get(url=status_url, auth=(self.admin_user, self.admin_pass))
+ response = r.json()
+ return response['status']
+ except requests.exceptions.RequestException as e:
+ self.logger.error('Failed to get dedup status, %s', e)
+ # exit_on_error()
+
+ def start_dedup(self, app_id):
+ body = ""
+ try:
+ r = requests.post(url=self.get_dedup_url() + "/" + app_id, data=body, auth=(self.admin_user, self.admin_pass))
+ if r.status_code == 200:
+ response = r.json()
+ return response['jobId']
+ else:
+ self.logger.error('Failed to start dedup, %s', r)
+ exit_on_error(str(r))
+
+ except requests.exceptions.RequestException as e:
+ self.logger.error('Unable to make API request for dedup, %s', e)
+ exit_on_error(str(e))
+
+ def is_dedup_running(self, job):
+ status = self.get_dedup_status(job)
+ self.logger.info('Dedup status=[%s]', status)
+ if status != "COMPLETE":
+ return True
+ else:
+ return False
+
def is_endpoint_available(self):
try:
@@ -490,6 +603,24 @@ class Migrate:
)
+ def get_app_ids(self):
+
+ try:
+ url = self.endpoint + "/management/orgs/" + self.org + "/apps"
+ r = requests.get(url=url)
+ if r.status_code != 200:
+ exit_on_error('Database Bootstrap Failed')
+
+ apps = r.json()["data"]
+ app_ids = []
+ for appId in apps.values():
+ app_ids.append(appId)
+
+ return app_ids;
+
+ except requests.exceptions.RequestException as e:
+ self.logger.error('Unable to get list of application ids, %s', e)
+ exit_on_error(str(e))
def get_current_time():
return str(int(time.time()*1000))
[23/50] [abbrv] usergrid git commit: Added --super and --admin so we
can specify both superuser and org admin credentials. Did some testing and
migration seems to work for one tenant,
have not tried two yet. Commented out the de-dup calls because de-dup
Posted by sn...@apache.org.
Added --super and --admin so we can specify both superuser and org admin credentials.
Did some testing and migration seems to work for one tenant, have not tried two yet.
Commented out the de-dup calls because de-dup never finishes.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/471dc359
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/471dc359
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/471dc359
Branch: refs/heads/asf-site
Commit: 471dc359d5920ea521c4058e7829a37494a17041
Parents: d79bf4c
Author: Dave Johnson <sn...@apache.org>
Authored: Thu Oct 29 10:07:33 2015 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Thu Oct 29 10:07:33 2015 -0400
----------------------------------------------------------------------
stack/scripts/migrate_entity_data.py | 130 ++++++++++++++++++++----------
1 file changed, 87 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/471dc359/stack/scripts/migrate_entity_data.py
----------------------------------------------------------------------
diff --git a/stack/scripts/migrate_entity_data.py b/stack/scripts/migrate_entity_data.py
index f576108..0edd319 100644
--- a/stack/scripts/migrate_entity_data.py
+++ b/stack/scripts/migrate_entity_data.py
@@ -20,7 +20,9 @@
#
# STEP 1 - SETUP TENANT ONE TOMCAT RUNNING 2.1 NOT IN SERVICE AND INIT MIGRATION
#
-# python migrate_entity_data.py --org <org1name> --user <superuser>:<superpass> --init
+# Login to the Tomcat instance and run this command, specifying both superuser and tenant organization admin creds:
+#
+# python migrate_entity_data.py --org <org1name> --super <user>:<pass> --admin <user>:<pass> --init
#
# This command will setup and bootstrap the database, setup the migration system and update index mappings:
# - /system/database/setup
@@ -31,24 +33,34 @@
# Then it will migrate appinfos, re-index the management app and then for each of the specified org's apps
# it will de-dup connections and re-index the app.
#
+# Write down the 'Re-index start' timestamp when this is finished.
+#
# STEP 2 - PUT TENANT ONE TOMCATS IN SERVICE AND DO DELTA MIGRATION
#
-# python migrate_entity_data.py --org <org1name> --user <superuser>:<superpass> --date
+# On the same Tomcat instance and run this command with the --date timestamp you noted in the previous step:
+#
+# python migrate_entity_data.py --org <org1name> --super <user>:<pass> --admin <user>:<pass> --date <timestamp>
#
# Then it will migrate appinfos, re-index the management app and then for each of the specified org's apps
# it will de-dup connections and re-index the app with a start-date specified so only data modified since
# STEP 1 will be re-indexed.
#
-# STEP 3 - SETUP TENENT TWO TOMCAT RUNNING 2.1 NOT IN SERVICE
+# STEP 3 - SETUP TENANT TWO TOMCAT RUNNING 2.1 NOT IN SERVICE
+#
+# Login to the Tomcat instance and run this command, specifying both superuser and tenant organization admin creds:
#
-# python migrate_entity_data.py --org <org2name> --user <superuser>:<superpass> --date
+# python migrate_entity_data.py --org <org2name> --super <user>:<pass> --admin <user>:<pass>
#
# This command will migrate appinfos, re-index the management app and then for each of the specified org's apps
# it will de-dup connections and re-index the app.
#
+# Write down the 'Re-index start' timestamp when this is finished.
+
# STEP 4 - PUT TENANT TWO TOMCATS IN SERVICE AND DO DELTA MIGRATION
#
-# python migrate_entity_data.py --org <org2name> --user <superuser>:<superpass> --date
+# On the same Tomcat instance and run this command with the --date timestamp you noted in the previous step:
+#
+# python migrate_entity_data.py --org <org2name> --super <user>:<pass> --admin <user>:<pass> --date <timestamp>
#
# Then it will migrate appinfos, re-index the management app and then for each of the specified org's apps
# it will de-dup connections and re-index the app with a start-date specified so only data modified since
@@ -56,7 +68,9 @@
#
# STEP 5 - FULL DATA MIGRATION
#
-# python migrate_entity_data.py --user <superuser>:<superpass> --full
+# Login to any Tomcat instance in the cluster and run this command (admin user creds must be specificed but will be ignored):
+#
+# python migrate_entity_data.py --super <user>:<pass> --admin <user>:<pass> --full
#
# This command will run the full data migration.
#
@@ -87,6 +101,8 @@ PLUGIN_ENTITYDATA = 'collections-entity-data'
PLUGIN_INDEX_MAPPING = 'index_mapping_migration'
PLUGIN_CORE_DATA = 'core-data'
+MANAGEMENT_APP_ID = 'b6768a08-b5d5-11e3-a495-11ddb1de66c8'
+
def parse_args():
@@ -97,8 +113,13 @@ def parse_args():
type=str,
default='http://localhost:8080')
- parser.add_argument('--user',
- help='System Admin Credentials used to authenticate with Usergrid <user:pass>',
+ parser.add_argument('--super',
+ help='Superuser username and creds <user:pass>',
+ type=str,
+ required=True)
+
+ parser.add_argument('--admin',
+ help='Organization admin creds <user:pass>',
type=str,
required=True)
@@ -124,13 +145,22 @@ def parse_args():
my_args = parser.parse_args(sys.argv[1:])
arg_vars = vars(my_args)
- creds = arg_vars['user'].split(':')
+
+ creds = arg_vars['super'].split(':')
+ if len(creds) != 2:
+ print('Superuser credentials not properly specified. Must be "-u <user:pass>". Exiting...')
+ exit_on_error()
+ else:
+ arg_vars['superuser'] = creds[0]
+ arg_vars['superpass'] = creds[1]
+
+ creds = arg_vars['super'].split(':')
if len(creds) != 2:
- print('Credentials not properly specified. Must be "-u <user:pass>". Exiting...')
+ print('Org admin credentials not properly specified. Must be "-u <user:pass>". Exiting...')
exit_on_error()
else:
- arg_vars['user'] = creds[0]
- arg_vars['pass'] = creds[1]
+ arg_vars['adminuser'] = creds[0]
+ arg_vars['adminpass'] = creds[1]
return arg_vars
@@ -148,8 +178,10 @@ class Migrate:
'full_data_migration_start': '',
'full_data_migration_end': ''}
self.logger = init_logging(self.__class__.__name__)
- self.admin_user = self.args['user']
- self.admin_pass = self.args['pass']
+ self.super_user = self.args['superuser']
+ self.super_pass = self.args['superpass']
+ self.admin_user = self.args['adminuser']
+ self.admin_pass = self.args['adminpass']
self.org = self.args['org']
self.init = self.args['init']
self.full = self.args['full']
@@ -241,7 +273,7 @@ class Migrate:
# Reindex management app
- job = self.start_reindex()
+ job = self.start_app_reindex(MANAGEMENT_APP_ID)
self.metrics['reindex_start'] = get_current_time()
self.logger.info('Started Re-index. Job=[%s]', job)
is_running = True
@@ -261,18 +293,18 @@ class Migrate:
for app_id in app_ids:
# De-dep app
- job = self.start_app_dedup(app_id)
- self.metrics['dedup_start_' + app_id] = get_current_time()
- self.logger.info('Started dedup. App=[%s], Job=[%s]', app_id, job)
- is_running = True
- while is_running:
- time.sleep(STATUS_INTERVAL_SECONDS)
- is_running = self.is_reindex_running(job)
- if not is_running:
- break
-
- self.logger.info("Finished dedup. App=[%s], Job=[%s]", app_id, job)
- self.metrics['dedup_end_' + app_id] = get_current_time()
+ # job = self.start_dedup(app_id)
+ # self.metrics['dedup_start_' + app_id] = get_current_time()
+ # self.logger.info('Started dedup. App=[%s], Job=[%s]', app_id, job)
+ # is_running = True
+ # while is_running:
+ # time.sleep(STATUS_INTERVAL_SECONDS)
+ # is_running = self.is_dedup_running(job)
+ # if not is_running:
+ # break
+ #
+ # self.logger.info("Finished dedup. App=[%s], Job=[%s]", app_id, job)
+ # self.metrics['dedup_end_' + app_id] = get_current_time()
# Re-index app
job = self.start_app_reindex(app_id)
@@ -330,7 +362,7 @@ class Migrate:
def start_core_data_migration(self):
try:
- r = requests.put(url=self.get_migration_url(), auth=(self.admin_user, self.admin_pass))
+ r = requests.put(url=self.get_migration_url(), auth=(self.admin_user, self.super_pass))
response = r.json()
return response
except requests.exceptions.RequestException as e:
@@ -339,7 +371,7 @@ class Migrate:
def start_fulldata_migration(self):
try:
- r = requests.put(url=self.get_migration_url(), auth=(self.admin_user, self.admin_pass))
+ r = requests.put(url=self.get_migration_url(), auth=(self.admin_user, self.super_pass))
response = r.json()
return response
except requests.exceptions.RequestException as e:
@@ -350,7 +382,7 @@ class Migrate:
try:
# TODO fix this URL
migrateUrl = self.get_migration_url() + '/' + PLUGIN_MIGRATION_SYSTEM
- r = requests.put(url=migrateUrl, auth=(self.admin_user, self.admin_pass))
+ r = requests.put(url=migrateUrl, auth=(self.admin_user, self.super_pass))
response = r.json()
return response
except requests.exceptions.RequestException as e:
@@ -360,7 +392,7 @@ class Migrate:
def run_database_setup(self):
try:
setupUrl = self.get_database_setup_url()
- r = requests.put(url=setupUrl, auth=(self.admin_user, self.admin_pass))
+ r = requests.put(url=setupUrl, auth=(self.super_user, self.super_pass))
if r.status_code != 200:
exit_on_error('Database Setup Failed')
@@ -371,7 +403,7 @@ class Migrate:
def run_database_bootstrap(self):
try:
setupUrl = self.get_database_bootstrap_url()
- r = requests.put(url=setupUrl, auth=(self.admin_user, self.admin_pass))
+ r = requests.put(url=setupUrl, auth=(self.super_user, self.super_pass))
if r.status_code != 200:
exit_on_error('Database Bootstrap Failed')
@@ -382,7 +414,7 @@ class Migrate:
def start_index_mapping_migration(self):
try:
migrateUrl = self.get_migration_url() + '/' + PLUGIN_INDEX_MAPPING
- r = requests.put(url=migrateUrl, auth=(self.admin_user, self.admin_pass))
+ r = requests.put(url=migrateUrl, auth=(self.super_user, self.super_pass))
response = r.json()
return response
except requests.exceptions.RequestException as e:
@@ -403,7 +435,7 @@ class Migrate:
version = TARGET_ENTITY_DATA_VERSION - 1
body = json.dumps({PLUGIN_ENTITYDATA: version, PLUGIN_APPINFO: version})
try:
- r = requests.put(url=self.get_reset_migration_url(), data=body, auth=(self.admin_user, self.admin_pass))
+ r = requests.put(url=self.get_reset_migration_url(), data=body, auth=(self.super_user, self.super_pass))
response = r.json()
self.logger.info('Resetting data migration versions to %s=[%s] '
'and %s=[%s]', PLUGIN_ENTITYDATA, version, PLUGIN_APPINFO, version)
@@ -416,7 +448,7 @@ class Migrate:
version = TARGET_APPINFO_VERSION - 1
body = json.dumps({PLUGIN_APPINFO: version})
try:
- r = requests.put(url=self.get_reset_migration_url(), data=body, auth=(self.admin_user, self.admin_pass))
+ r = requests.put(url=self.get_reset_migration_url(), data=body, auth=(self.super_user, self.super_pass))
response = r.json()
self.logger.info('Resetting appinfo migration versions to %s=[%s]', PLUGIN_APPINFO, version)
return response
@@ -495,7 +527,7 @@ class Migrate:
def check_data_migration_status(self):
try:
- r = requests.get(url=self.get_migration_status_url(), auth=(self.admin_user, self.admin_pass))
+ r = requests.get(url=self.get_migration_status_url(), auth=(self.super_user, self.super_pass))
if r.status_code == 200:
response = r.json()
return response
@@ -510,7 +542,7 @@ class Migrate:
status_url = self.get_reindex_url()+'/' + job
try:
- r = requests.get(url=status_url, auth=(self.admin_user, self.admin_pass))
+ r = requests.get(url=status_url, auth=(self.super_user, self.super_pass))
response = r.json()
return response['status']
except requests.exceptions.RequestException as e:
@@ -523,7 +555,7 @@ class Migrate:
body = json.dumps({'updated': self.start_date})
try:
- r = requests.post(url=self.get_reindex_url(), data=body, auth=(self.admin_user, self.admin_pass))
+ r = requests.post(url=self.get_reindex_url(), data=body, auth=(self.super_user, self.super_pass))
if r.status_code == 200:
response = r.json()
@@ -546,7 +578,8 @@ class Migrate:
def get_dedup_status(self, job):
status_url = self.get_dedup_url()+'/' + job
try:
- r = requests.get(url=status_url, auth=(self.admin_user, self.admin_pass))
+ r = requests.get(url=status_url, auth=(self.super_user, self.super_pass))
+ print r.text
response = r.json()
return response['status']
except requests.exceptions.RequestException as e:
@@ -556,10 +589,11 @@ class Migrate:
def start_dedup(self, app_id):
body = ""
try:
- r = requests.post(url=self.get_dedup_url() + "/" + app_id, data=body, auth=(self.admin_user, self.admin_pass))
+ r = requests.post(url=self.get_dedup_url() + "/" + app_id, data=body, auth=(self.super_user, self.super_pass))
if r.status_code == 200:
response = r.json()
- return response['jobId']
+ print r.text
+ return response['status']['jobStatusId']
else:
self.logger.error('Failed to start dedup, %s', r)
exit_on_error(str(r))
@@ -606,10 +640,20 @@ class Migrate:
def get_app_ids(self):
try:
- url = self.endpoint + "/management/orgs/" + self.org + "/apps"
+
+ url = self.endpoint + "/management/token"
+ body = json.dumps({"grant_type":"password","username":self.admin_user,"password":self.admin_pass})
+ r = requests.post(url=url, data=body)
+ if ( r.status_code != 200 ):
+ print "Error logging in: " + r.text
+ return
+
+ access_token = r.json()["access_token"]
+
+ url = self.endpoint + "/management/orgs/" + self.org + "/apps?access_token=" + access_token
r = requests.get(url=url)
if r.status_code != 200:
- exit_on_error('Database Bootstrap Failed')
+ exit_on_error('Cannot get app ids: ' + r.text)
apps = r.json()["data"]
app_ids = []
[08/50] [abbrv] usergrid git commit: USERGRID-1044: attempt #2 to
preprocess response body
Posted by sn...@apache.org.
USERGRID-1044: attempt #2 to preprocess response body
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/4b2d0035
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/4b2d0035
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/4b2d0035
Branch: refs/heads/asf-site
Commit: 4b2d00355e5645ab5c158b4c4f8e228037879af4
Parents: 2778f0a
Author: Mike Dunker <md...@apigee.com>
Authored: Tue Oct 27 11:29:16 2015 -0700
Committer: Mike Dunker <md...@apigee.com>
Committed: Tue Oct 27 11:29:16 2015 -0700
----------------------------------------------------------------------
.../usergrid/scenarios/EntityCollectionScenarios.scala | 13 ++++---------
1 file changed, 4 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/4b2d0035/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
index bcdb49a..f37a2d2 100644
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
@@ -223,15 +223,10 @@ object EntityCollectionScenarios {
.transformResponse {
case response if response.isReceived =>
new ResponseWrapper(response) {
- override val body = {
- val contentType = response.header("content-type").getOrElse("").toLowerCase
- if (contentType.contains("json")) {
- StringResponseBody(response.body.string, response.charset)
- } else {
- StringResponseBody("{}", StandardCharsets.UTF_8)
- }
- }
- }
+ val contentType = response.header("content-type").getOrElse("").toLowerCase
+ val bodyStr = if (contentType.contains("json")) response.body.string else "[]"
+ override val body = StringResponseBody(bodyStr, response.charset)
+ }
}
// 200 for success, 400 if already exists
.check(status.saveAs(SessionVarStatus), extractEntityUuid(SessionVarUuid), extractEntityModified(SessionVarModified)))
[05/50] [abbrv] usergrid git commit: USERGRID-1044: make jsonPath
extraction optional to keep users from exiting
Posted by sn...@apache.org.
USERGRID-1044: make jsonPath extraction optional to keep users from exiting
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/3c60c49a
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/3c60c49a
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/3c60c49a
Branch: refs/heads/asf-site
Commit: 3c60c49a157be63ec626ff88666472f2d4876554
Parents: 1a6de02
Author: Mike Dunker <md...@apigee.com>
Authored: Tue Oct 27 08:52:17 2015 -0700
Committer: Mike Dunker <md...@apigee.com>
Committed: Tue Oct 27 08:52:17 2015 -0700
----------------------------------------------------------------------
.../org/apache/usergrid/helpers/Extractors.scala | 18 +++++++++++-------
1 file changed, 11 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3c60c49a/stack/loadtests/src/main/scala/org/apache/usergrid/helpers/Extractors.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/helpers/Extractors.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/helpers/Extractors.scala
index 136a561..b21f9d7 100644
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/helpers/Extractors.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/helpers/Extractors.scala
@@ -37,7 +37,7 @@ object Extractors {
jsonPath("$.cursor").transformOption(extract => {
//it may or may not be present. If it is, save it, otherwise save it as an empty string
extract.orElse(Some(""))
- }).saveAs(saveAsName)
+ }).optional.saveAs(saveAsName)
}
/**
@@ -47,7 +47,7 @@ object Extractors {
jsonPath("$.entities[0].uuid").transformOption(extract => {
//it may or may not be present. If it is, save it, otherwise save it as an empty string
extract.orElse(Some(""))
- }).saveAs(saveAsName)
+ }).optional.saveAs(saveAsName)
}
/**
@@ -57,7 +57,7 @@ object Extractors {
jsonPath("$.entities[0].name").transformOption(extract => {
//it may or may not be present. If it is, save it, otherwise save it as an empty string
extract.orElse(Some(""))
- }).saveAs(saveAsName)
+ }).optional.saveAs(saveAsName)
}
/**
@@ -67,21 +67,25 @@ object Extractors {
jsonPath("$.entities[0].modified").ofType[Long].transformOption(extract => {
//it may or may not be present. If it is, save it, otherwise save it as -1
extract.orElse(Some(-1))
- }).saveAs(saveAsName)
+ }).optional.saveAs(saveAsName)
}
/**
* Will extract the audit entities from the get collection response.
*/
def extractAuditEntities(saveAsName: String) = {
- jsonPath("$.entities[*]").ofType[Map[String,Any]].findAll.transformOption(extract => { extract.orElse(Some(Seq.empty)) }).saveAs(saveAsName)
+ jsonPath("$.entities[*]").ofType[Map[String,Any]].findAll.transformOption(extract => {
+ extract.orElse(Some(Seq.empty))
+ }).optional.saveAs(saveAsName)
}
/**
* Will extract the audit entities from the get collection response.
*/
def extractAuditEntity(saveAsName: String) = {
- jsonPath("$.entities[0]").ofType[Map[String,Any]].findAll.transformOption(extract => { extract.orElse(Some(Seq.empty)) }).saveAs(saveAsName)
+ jsonPath("$.entities[0]").ofType[Map[String,Any]].findAll.transformOption(extract => {
+ extract.orElse(Some(Seq.empty))
+ }).optional.saveAs(saveAsName)
}
/**
@@ -100,7 +104,7 @@ object Extractors {
def maybeExtractEntities(saveAsName: String) = {
jsonPath("$.entities").ofType[Seq[Any]].transformOption(extract => {
extract.orElse(Some(Seq()))
- }).saveAs(saveAsName)
+ }).optional.saveAs(saveAsName)
}
/**
[07/50] [abbrv] usergrid git commit: Merge commit
'refs/pull/414/head' of github.com:apache/usergrid into 2.1-release
Posted by sn...@apache.org.
Merge commit 'refs/pull/414/head' of github.com:apache/usergrid into 2.1-release
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/e018c1e0
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/e018c1e0
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/e018c1e0
Branch: refs/heads/asf-site
Commit: e018c1e0d65e3af339e36c69a4e59fe2d2a83167
Parents: 2aec8a0 7a4f36a
Author: Michael Russo <mi...@gmail.com>
Authored: Tue Oct 27 10:54:04 2015 -0700
Committer: Michael Russo <mi...@gmail.com>
Committed: Tue Oct 27 10:54:04 2015 -0700
----------------------------------------------------------------------
.../service/StatusServiceImpl.java | 3 +-
.../usergrid/rest/ApplicationsResource.java | 178 ----------
.../apache/usergrid/rest/DatabaseResource.java | 98 ------
.../org/apache/usergrid/rest/IndexResource.java | 328 ------------------
.../apache/usergrid/rest/MigrateResource.java | 268 ---------------
.../apache/usergrid/rest/SystemResource.java | 103 ------
.../rest/system/ApplicationsResource.java | 178 ++++++++++
.../rest/system/ConnectionResource.java | 202 +++++++++++
.../usergrid/rest/system/DatabaseResource.java | 100 ++++++
.../usergrid/rest/system/IndexResource.java | 331 +++++++++++++++++++
.../usergrid/rest/system/MigrateResource.java | 270 +++++++++++++++
.../usergrid/rest/system/SystemResource.java | 108 ++++++
12 files changed, 1191 insertions(+), 976 deletions(-)
----------------------------------------------------------------------
[35/50] [abbrv] usergrid git commit: Addresses comments in review
Posted by sn...@apache.org.
Addresses comments in review
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/3d120578
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/3d120578
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/3d120578
Branch: refs/heads/asf-site
Commit: 3d1205784528b5026695910943243a7a38fcbdde
Parents: ba629da
Author: Todd Nine <tn...@apigee.com>
Authored: Fri Oct 30 10:51:58 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Fri Oct 30 10:51:58 2015 -0600
----------------------------------------------------------------------
.../apache/usergrid/rest/applications/users/UserResource.java | 6 +++---
.../rest/applications/collection/users/UserResourceIT.java | 2 --
2 files changed, 3 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3d120578/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java
index 1116469..fb2962e 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java
@@ -173,11 +173,11 @@ public class UserResource extends ServiceResource {
public JSONWithPadding getUserCredentials(@QueryParam("callback") @DefaultValue("callback") String callback )
throws Exception {
- logger.info( "UserResource.setUserPassword" );
+ logger.info( "UserResource.getUserCredentials" );
final ApiResponse response = createApiResponse();
- response.setAction( "get user password" );
+ response.setAction( "get user credentials" );
final UUID applicationId = getApplicationId();
final UUID targetUserId = getUserUuid();
@@ -210,7 +210,7 @@ public class UserResource extends ServiceResource {
@QueryParam("callback") @DefaultValue("callback") String callback )
throws Exception {
- logger.info( "UserResource.setUserPassword" );
+ logger.info( "UserResource.setUserCredentials" );
if ( json == null ) {
return null;
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3d120578/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/UserResourceIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/UserResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/UserResourceIT.java
index deda5eb..f258f94 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/UserResourceIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/UserResourceIT.java
@@ -1154,7 +1154,6 @@ public class UserResourceIT extends AbstractRestIT {
//here we modify the hash a little, this way we can break password validation, then re-set it to ensure we're actually updating the credentials info correctly.
final String borkedSecret = originalSecret.substring( 0, originalSecret.length() -1 );
- credentials.put( "credentials", borkedSecret );
credentials.put( "secret", borkedSecret );
//now PUT it
@@ -1192,7 +1191,6 @@ public class UserResourceIT extends AbstractRestIT {
//now put the correct one
- credentials.put( "credentials", originalSecret );
credentials.put( "secret", originalSecret );
[45/50] [abbrv] usergrid git commit: This closes #134 - cannot merge
from unknown repository
Posted by sn...@apache.org.
This closes #134 - cannot merge from unknown repository
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/11594ceb
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/11594ceb
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/11594ceb
Branch: refs/heads/asf-site
Commit: 11594cebc32bbcb14de72da8d500b3ba92684c99
Parents: 02d6a91
Author: Mike Dunker <md...@apigee.com>
Authored: Tue Nov 3 14:51:42 2015 -0800
Committer: Mike Dunker <md...@apigee.com>
Committed: Tue Nov 3 14:51:42 2015 -0800
----------------------------------------------------------------------
----------------------------------------------------------------------
[12/50] [abbrv] usergrid git commit: Updates the message flow to
allow for multiple processor threads per SQS take thread
Posted by sn...@apache.org.
Updates the message flow to allow for multiple processor threads per SQS take thread
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/76476f17
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/76476f17
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/76476f17
Branch: refs/heads/asf-site
Commit: 76476f17cf8e8be6f01660db3d21110eda8247f5
Parents: 1b43bda
Author: Todd Nine <tn...@apigee.com>
Authored: Tue Oct 27 14:35:34 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Tue Oct 27 14:35:34 2015 -0600
----------------------------------------------------------------------
.../asyncevents/AmazonAsyncEventService.java | 68 +++++++++++---------
.../index/IndexProcessorFig.java | 13 +++-
2 files changed, 51 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/76476f17/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
index d93e304..6b9abbc 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
@@ -679,35 +679,45 @@ public class AmazonAsyncEventService implements AsyncEventService {
}
while ( true );
}
- } )
- //this won't block our read loop, just reads and proceeds
- .map( messages -> {
- if ( messages == null || messages.size() == 0 ) {
- return null;
- }
-
- try {
- List<IndexEventResult> indexEventResults = callEventHandlers( messages );
- List<QueueMessage> messagesToAck = submitToIndex( indexEventResults );
- if ( messagesToAck == null || messagesToAck.size() == 0 ) {
- logger.error( "No messages came back from the queue operation should have seen "
- + messages.size(), messages );
- return messagesToAck;
- }
- if ( messagesToAck.size() < messages.size() ) {
- logger.error( "Missing messages from queue post operation", messages,
- messagesToAck );
- }
- //ack each message, but only if we didn't error.
- ack( messagesToAck );
- return messagesToAck;
- }
- catch ( Exception e ) {
- logger.error( "failed to ack messages to sqs", e );
- return null;
- //do not rethrow so we can process all of them
- }
- } );
+ } ) //this won't block our read loop, just reads and proceeds
+ .flatMap( sqsMessages -> {
+
+ //do this on a different schedule, and introduce concurrency with flatmap for faster processing
+ return Observable.just( sqsMessages )
+
+ .map( messages -> {
+ if ( messages == null || messages.size() == 0 ) {
+ return null;
+ }
+
+ try {
+ List<IndexEventResult> indexEventResults =
+ callEventHandlers( messages );
+ List<QueueMessage> messagesToAck =
+ submitToIndex( indexEventResults );
+ if ( messagesToAck == null || messagesToAck.size() == 0 ) {
+ logger.error(
+ "No messages came back from the queue operation should "
+ + "have seen "
+ + messages.size(), messages );
+ return messagesToAck;
+ }
+ if ( messagesToAck.size() < messages.size() ) {
+ logger.error( "Missing messages from queue post operation",
+ messages, messagesToAck );
+ }
+ //ack each message, but only if we didn't error.
+ ack( messagesToAck );
+ return messagesToAck;
+ }
+ catch ( Exception e ) {
+ logger.error( "failed to ack messages to sqs", e );
+ return null;
+ //do not rethrow so we can process all of them
+ }
+ } ).subscribeOn( rxTaskScheduler.getAsyncIOScheduler() );
+ //end flatMap
+ }, indexProcessorFig.getEventConcurrencyFactor() );
//start in the background
http://git-wip-us.apache.org/repos/asf/usergrid/blob/76476f17/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java
index 7650c62..9d02717 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java
@@ -36,6 +36,8 @@ public interface IndexProcessorFig extends GuicyFig {
String ELASTICSEARCH_WORKER_COUNT = "elasticsearch.worker_count";
+ String EVENT_CONCURRENCY_FACTOR = "event.concurrency.factor";
+
String ELASTICSEARCH_QUEUE_IMPL = "elasticsearch.queue_impl";
String INDEX_QUEUE_READ_TIMEOUT = "elasticsearch.queue_read_timeout";
@@ -70,9 +72,18 @@ public interface IndexProcessorFig extends GuicyFig {
int getIndexQueueVisibilityTimeout();
/**
+ * The number of worker threads used when handing off messages from the SQS thread
+ */
+ @Default( "20" )
+ @Key( EVENT_CONCURRENCY_FACTOR )
+ int getEventConcurrencyFactor();
+
+
+
+ /**
* The number of worker threads used to read index write requests from the queue.
*/
- @Default( "16" )
+ @Default( "8" )
@Key( ELASTICSEARCH_WORKER_COUNT )
int getWorkerCount();
[17/50] [abbrv] usergrid git commit: USERGRID-1044: add shutdown
handler to audit simulations
Posted by sn...@apache.org.
USERGRID-1044: add shutdown handler to audit simulations
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/2e70a72b
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/2e70a72b
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/2e70a72b
Branch: refs/heads/asf-site
Commit: 2e70a72b99ac0a48e5f703a1b464fdad3c4d2665
Parents: fc23b79
Author: Mike Dunker <md...@apigee.com>
Authored: Tue Oct 27 16:45:02 2015 -0700
Committer: Mike Dunker <md...@apigee.com>
Committed: Tue Oct 27 16:45:02 2015 -0700
----------------------------------------------------------------------
.../apache/usergrid/simulations/AuditSimulation.scala | 11 +++++++++++
.../usergrid/simulations/ConfigurableSimulation.scala | 2 +-
2 files changed, 12 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/2e70a72b/stack/loadtests/src/main/scala/org/apache/usergrid/simulations/AuditSimulation.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/simulations/AuditSimulation.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/simulations/AuditSimulation.scala
index 47323ea..bdec3dd 100755
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/simulations/AuditSimulation.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/simulations/AuditSimulation.scala
@@ -57,6 +57,10 @@ class AuditSimulation extends Simulation {
}
after {
+ endHandler
+ }
+
+ def endHandler: Unit = {
Settings.setTestEndTime()
if (Settings.captureAuditUuids) {
val uuidDesc = Settings.scenarioType match {
@@ -69,5 +73,12 @@ class AuditSimulation extends Simulation {
Settings.printAuditResults()
}
+ def abortedEarly: Unit = {
+ println(">>>>>>>>>>>>AUDIT ABORTED")
+ endHandler
+ }
+
+ sys addShutdownHook abortedEarly
+
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/2e70a72b/stack/loadtests/src/main/scala/org/apache/usergrid/simulations/ConfigurableSimulation.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/simulations/ConfigurableSimulation.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/simulations/ConfigurableSimulation.scala
index 9d4b05d..a3eb595 100755
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/simulations/ConfigurableSimulation.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/simulations/ConfigurableSimulation.scala
@@ -99,7 +99,7 @@ class ConfigurableSimulation extends Simulation {
Settings.printSettingsSummary(true)
}
- sys addShutdownHook(endHandler)
+ sys addShutdownHook endHandler
}
[27/50] [abbrv] usergrid git commit: Fixes GET + PUT for credentials
info
Posted by sn...@apache.org.
Fixes GET + PUT for credentials info
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/4784b34f
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/4784b34f
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/4784b34f
Branch: refs/heads/asf-site
Commit: 4784b34f3342b32639ee23beb9ccafa9a5443ed0
Parents: 7fc7c55
Author: Todd Nine <tn...@apigee.com>
Authored: Thu Oct 29 12:37:31 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Thu Oct 29 14:42:41 2015 -0600
----------------------------------------------------------------------
.../usergrid/persistence/CredentialsInfo.java | 46 ++++++++++++++++++++
.../rest/applications/users/UserResource.java | 11 +++--
2 files changed, 53 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/4784b34f/stack/core/src/main/java/org/apache/usergrid/persistence/CredentialsInfo.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/CredentialsInfo.java b/stack/core/src/main/java/org/apache/usergrid/persistence/CredentialsInfo.java
index 957f8a8..c7c4cba 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/CredentialsInfo.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/CredentialsInfo.java
@@ -20,7 +20,10 @@ package org.apache.usergrid.persistence;
import com.fasterxml.jackson.annotation.JsonAnyGetter;
import com.fasterxml.jackson.annotation.JsonAnySetter;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
+import com.google.common.base.Preconditions;
+
import java.io.Serializable;
+import java.util.List;
import java.util.Map;
import java.util.TreeMap;
import javax.xml.bind.annotation.XmlRootElement;
@@ -118,6 +121,11 @@ public class CredentialsInfo implements Comparable<CredentialsInfo>,Serializable
}
+ public void setCreated( final Long created ) {
+ this.created = created;
+ }
+
+
@JsonAnySetter
public void setProperty( String key, Object value ) {
properties.put( key, value );
@@ -172,4 +180,42 @@ public class CredentialsInfo implements Comparable<CredentialsInfo>,Serializable
}
return o.created.compareTo( created );
}
+
+
+ /**
+ * Parse the json representation into an object
+ * @param json
+ * @return
+ */
+ public static CredentialsInfo fromJson(final Map<String, Object> json){
+ final boolean recoverable = ( boolean ) json.get( "recoverable");
+
+ final boolean encrypted = ( boolean ) json.get("encrypted");
+ final String hashType = ( String ) json.get( "hashType");
+ final long created = ( long ) json.get( "created");
+ final String secret = ( String ) json.get( "secret" );
+
+ final List<String> cryptoChain = ( List<String> ) json.get( "cryptoChain");
+
+ Preconditions.checkNotNull(created, "created is required");
+ Preconditions.checkNotNull(secret, "secret is required");
+ Preconditions.checkNotNull(cryptoChain, "cryptoChain is required");
+
+ Preconditions.checkArgument(cryptoChain.size() >= 1, "cryptoChain must have 1 or more entries");
+
+
+ final String[] cryptoString = new String[cryptoChain.size()];
+
+ cryptoChain.toArray( cryptoString );
+
+
+ final CredentialsInfo credentialsInfo = new CredentialsInfo();
+ credentialsInfo.setEncrypted( encrypted );
+ credentialsInfo.setHashType( hashType );
+ credentialsInfo.setCreated( created );
+ credentialsInfo.setSecret( secret );
+ credentialsInfo.setCryptoChain( cryptoString );
+
+ return credentialsInfo;
+ }
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/4784b34f/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java
index 716f367..1116469 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java
@@ -170,7 +170,7 @@ public class UserResource extends ServiceResource {
@GET
@RequireSystemAccess
@Path("credentials")
- public JSONWithPadding getUserPassword(@QueryParam("callback") @DefaultValue("callback") String callback )
+ public JSONWithPadding getUserCredentials(@QueryParam("callback") @DefaultValue("callback") String callback )
throws Exception {
logger.info( "UserResource.setUserPassword" );
@@ -204,6 +204,7 @@ public class UserResource extends ServiceResource {
@PUT
+ @RequireSystemAccess
@Path("credentials")
public JSONWithPadding setUserCredentials( @Context UriInfo ui, Map<String, Object> json,
@QueryParam("callback") @DefaultValue("callback") String callback )
@@ -217,13 +218,15 @@ public class UserResource extends ServiceResource {
ApiResponse response = createApiResponse();
response.setAction( "set user credentials" );
- Object credentials = json.get( "credentials" );
+ Map<String, Object> credentialsJson = ( Map<String, Object> ) json.get( "credentials" );
- if ( credentials == null ) {
+ if ( credentialsJson == null ) {
throw new IllegalArgumentException( "credentials sub object is required" );
}
+ final CredentialsInfo credentials = CredentialsInfo.fromJson( credentialsJson );
+
UUID applicationId = getApplicationId();
UUID targetUserId = getUserUuid();
@@ -233,7 +236,7 @@ public class UserResource extends ServiceResource {
}
- management.setAppUserCredentialsInfo( applicationId, targetUserId, ( CredentialsInfo ) credentials );
+ management.setAppUserCredentialsInfo( applicationId, targetUserId, credentials );
return new JSONWithPadding( response, callback );
[33/50] [abbrv] usergrid git commit: Add dup edges so we exercise the
dedup logic.
Posted by sn...@apache.org.
Add dup edges so we exercise the dedup logic.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/9ada6ea3
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/9ada6ea3
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/9ada6ea3
Branch: refs/heads/asf-site
Commit: 9ada6ea3732a3bfb72cc6a48d12353d0ac832ce6
Parents: b816a18
Author: Dave Johnson <sn...@apache.org>
Authored: Fri Oct 30 08:30:18 2015 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Fri Oct 30 08:30:18 2015 -0400
----------------------------------------------------------------------
stack/scripts/create_test_data.py | 4 ++++
1 file changed, 4 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/9ada6ea3/stack/scripts/create_test_data.py
----------------------------------------------------------------------
diff --git a/stack/scripts/create_test_data.py b/stack/scripts/create_test_data.py
index ef07254..3a913ca 100644
--- a/stack/scripts/create_test_data.py
+++ b/stack/scripts/create_test_data.py
@@ -155,6 +155,8 @@ class Creator:
targetType = appName + "_col1"
url = appUrl + "/" + sourceCollection + "/" + sourceName + "/has/" + targetType + "/" + targetName
r = requests.post(url=url + "?access_token=" + accessToken)
+ r = requests.post(url=url + "?access_token=" + accessToken)
+ r = requests.post(url=url + "?access_token=" + accessToken)
if ( r.status_code >= 400 ):
print "Error connecting entity " + sourceName + " to " + targetName + ": " + r.text
print "url is: " + url
@@ -168,6 +170,8 @@ class Creator:
targetType = appName + "_col2"
url = appUrl + "/" + sourceCollection + "/" + sourceName + "/has/" + targetType + "/" + targetName
r = requests.post(url=url + "?access_token=" + accessToken)
+ r = requests.post(url=url + "?access_token=" + accessToken)
+ r = requests.post(url=url + "?access_token=" + accessToken)
if ( r.status_code >= 400 ):
print "Error connecting entity " + sourceName + " to " + targetName + ": " + r.text
print "url is: " + url
[37/50] [abbrv] usergrid git commit: Merge commit
'refs/pull/418/head' of github.com:apache/usergrid into 2.1-release
Posted by sn...@apache.org.
Merge commit 'refs/pull/418/head' of github.com:apache/usergrid into 2.1-release
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/40dfa064
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/40dfa064
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/40dfa064
Branch: refs/heads/asf-site
Commit: 40dfa0647709881170ea0874036d0ea84875148f
Parents: 3d12057 9b589d3
Author: Michael Russo <mi...@gmail.com>
Authored: Fri Oct 30 12:33:48 2015 -0700
Committer: Michael Russo <mi...@gmail.com>
Committed: Fri Oct 30 12:33:48 2015 -0700
----------------------------------------------------------------------
.../rest/system/ConnectionResource.java | 48 +-
stack/scripts/create_test_data.py | 222 ++++++
stack/scripts/migrate_entity_data.py | 18 +-
stack/scripts/multitenant_migrate.py | 702 +++++++++++++++++++
4 files changed, 965 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
[48/50] [abbrv] usergrid git commit: add index test for geo
Posted by sn...@apache.org.
add index test for geo
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/a7b6a9cd
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/a7b6a9cd
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/a7b6a9cd
Branch: refs/heads/asf-site
Commit: a7b6a9cdcb0399d518365ec51d70874b7c37b92f
Parents: 38a7d62
Author: Shawn Feldman <sf...@apache.org>
Authored: Thu Nov 5 16:54:14 2015 -0700
Committer: Shawn Feldman <sf...@apache.org>
Committed: Thu Nov 5 16:54:14 2015 -0700
----------------------------------------------------------------------
.../usergrid/persistence/RebuildIndexTest.java | 118 ++++++++++++++++++-
1 file changed, 113 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a7b6a9cd/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java b/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java
index cea6ada..7d80644 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java
@@ -17,11 +17,7 @@
package org.apache.usergrid.persistence;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
+import java.util.*;
import org.apache.usergrid.corepersistence.index.IndexLocationStrategyFactory;
import org.junit.After;
@@ -300,6 +296,118 @@ public class RebuildIndexTest extends AbstractCoreIT {
readData( em, collectionName, ENTITIES_TO_INDEX, 3 );
}
+ @Test( timeout = 120000 )
+ public void rebuildIndexGeo() throws Exception {
+
+ logger.info( "Started rebuildIndex()" );
+
+ String rand = RandomStringUtils.randomAlphanumeric( 5 );
+ final UUID appId = setup.createApplication( "org_" + rand, "app_" + rand );
+
+ final EntityManager em = setup.getEmf().getEntityManager( appId );
+
+ final ReIndexService reIndexService = setup.getInjector().getInstance( ReIndexService.class );
+
+ // ----------------- create a bunch of entities
+
+ Map<String, Object> cat1map = new HashMap<String, Object>() {{
+ put( "name", "enzo" );
+ put( "color", "grey" );
+ put("location", new LinkedHashMap<String, Object>() {{
+ put("latitude", -35.746369);
+ put("longitude", 150.952183);
+ }});
+ }};
+ final double lat = -34.746369;
+ final double lon = 152.952183;
+ Map<String, Object> cat2map = new HashMap<String, Object>() {{
+ put( "name", "marquee" );
+ put( "color", "grey" );
+ put("location", new LinkedHashMap<String, Object>() {{
+ put("latitude", lat);
+ put("longitude", lon);
+ }});
+ }};
+ Map<String, Object> cat3map = new HashMap<String, Object>() {{
+ put( "name", "bertha" );
+ put( "color", "grey" );
+ put("location", new LinkedHashMap<String, Object>() {{
+ put("latitude", -33.746369);
+ put("longitude", 150.952183);
+ }});
+ }};
+
+ Entity cat1 = em.create( "cat", cat1map );
+ Entity cat2 = em.create( "cat", cat2map );
+ Entity cat3 = em.create( "cat", cat3map );
+
+
+ logger.info( "Created {} entities", ENTITIES_TO_INDEX );
+ app.refreshIndex();
+
+ // ----------------- test that we can read them, should work fine
+
+ logger.debug( "Read the data" );
+ final String collectionName = "cats";
+ Query q = Query.fromQL( "select * where color='grey'" ).withLimit( 1000 );
+ Results results = em.searchCollectionConsistent( em.getApplicationRef(), collectionName, q, 3 );
+ assertEquals(3,results.size());
+
+
+ // ----------------- delete the system and application indexes
+
+ logger.debug( "Deleting app index" );
+
+ deleteIndex( em.getApplicationId() );
+
+ // ----------------- test that we can read them, should fail
+
+ // deleting sytem app index will interfere with other concurrently running tests
+ //deleteIndex( CpNamingUtils.SYSTEM_APP_ID );
+
+ // ----------------- test that we can read them, should fail
+
+ logger.debug( "Reading data, should fail this time " );
+
+ results = em.searchCollectionConsistent( em.getApplicationRef(), collectionName, q, 0 );
+ assertEquals(results.size(),0);
+
+ // ----------------- rebuild index
+
+ logger.debug( "Preparing to rebuild all indexes" );
+
+
+ try {
+
+ final ReIndexRequestBuilder builder =
+ reIndexService.getBuilder().withApplicationId( em.getApplicationId() );
+
+ ReIndexService.ReIndexStatus status = reIndexService.rebuildIndex( builder );
+
+ assertNotNull( status.getJobId(), "JobId is present" );
+
+ logger.info( "Rebuilt index" );
+
+ waitForRebuild( status, reIndexService );
+
+ logger.info( "Rebuilt index" );
+
+ app.refreshIndex();
+ }
+ catch ( Exception ex ) {
+ logger.error( "Error rebuilding index", ex );
+ fail();
+ }
+
+ // ----------------- test that we can read them
+
+ Thread.sleep( 2000 );
+ results = em.searchCollectionConsistent( em.getApplicationRef(), collectionName, q, 3 );
+ assertEquals(results.size(),3);
+ q = Query.fromQL("select * where location within 100 of "+lat+", "+lon);
+ results = em.searchCollectionConsistent( em.getApplicationRef(), collectionName, q, 1 );
+ assertEquals(results.size(),1);
+ }
@Test( timeout = 120000 )
[26/50] [abbrv] usergrid git commit: Updated call for new read api in
2.0
Posted by sn...@apache.org.
Updated call for new read api in 2.0
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/7fc7c55b
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/7fc7c55b
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/7fc7c55b
Branch: refs/heads/asf-site
Commit: 7fc7c55b7839ff9ba5a71b1c3d74a095c27f49ff
Parents: 08e9781
Author: Todd Nine <tn...@apigee.com>
Authored: Thu Oct 29 13:56:14 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Thu Oct 29 13:56:14 2015 -0600
----------------------------------------------------------------------
.../apache/usergrid/rest/applications/users/UserResource.java | 4 ++--
.../java/org/apache/usergrid/management/ManagementService.java | 2 +-
.../usergrid/management/cassandra/ManagementServiceImpl.java | 4 ++--
3 files changed, 5 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/7fc7c55b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java
index fb10245..716f367 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java
@@ -169,7 +169,7 @@ public class UserResource extends ServiceResource {
@GET
@RequireSystemAccess
- @Path("password")
+ @Path("credentials")
public JSONWithPadding getUserPassword(@QueryParam("callback") @DefaultValue("callback") String callback )
throws Exception {
@@ -192,7 +192,7 @@ public class UserResource extends ServiceResource {
return new JSONWithPadding( response, callback );
}
- final CredentialsInfo credentialsInfo = management.getAppUserPasswordRaw( applicationId, targetUserId );
+ final CredentialsInfo credentialsInfo = management.getAppUserCredentialsInfo( applicationId, targetUserId );
response.setProperty( "credentials", credentialsInfo );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/7fc7c55b/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java b/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java
index 3f02e5a..cf2924b 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java
@@ -288,7 +288,7 @@ public interface ManagementService {
public void setAppUserPassword( UUID applicationId, UUID userId, String oldPassword, String newPassword )
throws Exception;
- CredentialsInfo getAppUserPasswordRaw( final UUID applicationId, final UUID userId ) throws Exception;
+ CredentialsInfo getAppUserCredentialsInfo( final UUID applicationId, final UUID userId ) throws Exception;
/**
* Set the credentials info into the
http://git-wip-us.apache.org/repos/asf/usergrid/blob/7fc7c55b/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
index b633727..dfd0cb1 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
@@ -2825,7 +2825,7 @@ public class ManagementServiceImpl implements ManagementService {
@Override
- public CredentialsInfo getAppUserPasswordRaw( final UUID applicationId, final UUID userId ) throws Exception {
+ public CredentialsInfo getAppUserCredentialsInfo( final UUID applicationId, final UUID userId ) throws Exception {
final User user = emf.getEntityManager( applicationId ).get( userId, User.class );
@@ -2833,7 +2833,7 @@ public class ManagementServiceImpl implements ManagementService {
throw new EntityNotFoundException("Could not find user with id " + userId + " in application" + applicationId );
}
- final CredentialsInfo ci = readUserPasswordCredentials( applicationId, userId );
+ final CredentialsInfo ci = readUserPasswordCredentials( applicationId, userId, User.ENTITY_TYPE );
if ( ci == null ) {
throw new EntityNotFoundException("Could not find credentials for user with id " + userId + " in application" + applicationId );
[06/50] [abbrv] usergrid git commit: USERGRID-1044: transform payload
to empty JSON if non-json
Posted by sn...@apache.org.
USERGRID-1044: transform payload to empty JSON if non-json
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/2778f0a0
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/2778f0a0
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/2778f0a0
Branch: refs/heads/asf-site
Commit: 2778f0a0b196bfc09e914033392ccb49271136e0
Parents: 3c60c49
Author: Mike Dunker <md...@apigee.com>
Authored: Tue Oct 27 10:49:44 2015 -0700
Committer: Mike Dunker <md...@apigee.com>
Committed: Tue Oct 27 10:49:44 2015 -0700
----------------------------------------------------------------------
.../org/apache/usergrid/helpers/Extractors.scala | 2 +-
.../scenarios/EntityCollectionScenarios.scala | 17 +++++++++++++++++
2 files changed, 18 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/2778f0a0/stack/loadtests/src/main/scala/org/apache/usergrid/helpers/Extractors.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/helpers/Extractors.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/helpers/Extractors.scala
index b21f9d7..4c6ac8d 100644
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/helpers/Extractors.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/helpers/Extractors.scala
@@ -85,7 +85,7 @@ object Extractors {
def extractAuditEntity(saveAsName: String) = {
jsonPath("$.entities[0]").ofType[Map[String,Any]].findAll.transformOption(extract => {
extract.orElse(Some(Seq.empty))
- }).optional.saveAs(saveAsName)
+ }).saveAs(saveAsName)
}
/**
http://git-wip-us.apache.org/repos/asf/usergrid/blob/2778f0a0/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
index ecd0c21..bcdb49a 100644
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
@@ -16,9 +16,13 @@
*/
package org.apache.usergrid.scenarios
+import java.nio.charset.StandardCharsets
+
import io.gatling.core.Predef._
import io.gatling.core.feeder.RecordSeqFeederBuilder
import io.gatling.http.Predef._
+import io.gatling.http.response.ResponseWrapper
+import io.gatling.http.response.StringResponseBody
import org.apache.usergrid.datagenerators.FeederGenerator
import org.apache.usergrid.enums.{CsvFeedPatternType, EndConditionType, AuthType}
import org.apache.usergrid.helpers.Extractors._
@@ -216,6 +220,19 @@ object EntityCollectionScenarios {
.headers(Headers.authToken)
.headers(Headers.usergridRegionHeaders)
.body(StringBody("""${entity}"""))
+ .transformResponse {
+ case response if response.isReceived =>
+ new ResponseWrapper(response) {
+ override val body = {
+ val contentType = response.header("content-type").getOrElse("").toLowerCase
+ if (contentType.contains("json")) {
+ StringResponseBody(response.body.string, response.charset)
+ } else {
+ StringResponseBody("{}", StandardCharsets.UTF_8)
+ }
+ }
+ }
+ }
// 200 for success, 400 if already exists
.check(status.saveAs(SessionVarStatus), extractEntityUuid(SessionVarUuid), extractEntityModified(SessionVarModified)))
.exec(session => {
[28/50] [abbrv] usergrid git commit: Adds test to prove functionality
of GET + PUT
Posted by sn...@apache.org.
Adds test to prove functionality of GET + PUT
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/ba629da5
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/ba629da5
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/ba629da5
Branch: refs/heads/asf-site
Commit: ba629da577235c0c9bedf4addcb44bfb5fdbdccb
Parents: 4784b34
Author: Todd Nine <tn...@apigee.com>
Authored: Thu Oct 29 15:25:42 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Thu Oct 29 15:25:42 2015 -0600
----------------------------------------------------------------------
.../collection/users/UserResourceIT.java | 109 +++++++++++++++++++
1 file changed, 109 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/ba629da5/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/UserResourceIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/UserResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/UserResourceIT.java
index 79b8c85..deda5eb 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/UserResourceIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/UserResourceIT.java
@@ -37,9 +37,15 @@ import org.apache.usergrid.rest.applications.utils.UserRepo;
import org.apache.usergrid.utils.UUIDUtils;
import com.sun.jersey.api.client.ClientResponse.Status;
+import com.sun.jersey.api.client.GenericType;
import com.sun.jersey.api.client.UniformInterfaceException;
+import com.sun.jersey.api.client.WebResource;
+import com.sun.jersey.api.client.filter.HTTPBasicAuthFilter;
+
import java.io.IOException;
+import javax.ws.rs.core.MediaType;
+
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
@@ -1103,4 +1109,107 @@ public class UserResourceIT extends AbstractRestIT {
assertEquals(response.getResponse().getEntities().get(0).get("uuid").toString(), userId.toString());
}
+
+
+
+ @Test
+ public void testCredentialsTransfer() throws Exception {
+
+ usersResource.post(new User("test_1", "Test1 User", "test_1@test.com", "test123")); // client.setApiUrl(apiUrl);
+ refreshIndex();
+
+ //Entity appInfo = this.app().get().getResponse().getEntities().get(0);
+
+ Token token = this.app().token().post(new Token("test_1", "test123"));
+
+ assertNotNull(token.getAccessToken());
+
+ final String superUserName = this.clientSetup.getSuperuserName();
+ final String superUserPassword = this.clientSetup.getSuperuserPassword();
+
+
+ //get the credentials info
+ final CollectionEndpoint collection = userResource.entity("test_1").collection( "credentials" );
+
+ final WebResource resource = collection.getResource();
+
+ resource.addFilter( new HTTPBasicAuthFilter(superUserName, superUserPassword) );
+
+
+
+ final ApiResponse response = resource.type( MediaType.APPLICATION_JSON_TYPE)
+ .accept( MediaType.APPLICATION_JSON ).get( org.apache.usergrid.rest.test.resource.model.ApiResponse.class );
+
+
+ //now get the credentials sub object
+
+ final Map<String, Object> credentials = ( Map<String, Object> ) response.getProperties().get( "credentials" );
+
+
+
+ //get out the hash and change it so we can validate
+ final String originalSecret = ( String ) credentials.get( "secret" );
+
+
+ //here we modify the hash a little, this way we can break password validation, then re-set it to ensure we're actually updating the credentials info correctly.
+ final String borkedSecret = originalSecret.substring( 0, originalSecret.length() -1 );
+
+ credentials.put( "credentials", borkedSecret );
+ credentials.put( "secret", borkedSecret );
+
+ //now PUT it
+
+
+ final Map<String, Map<String, Object>> wrapper = new HashMap<>( );
+ wrapper.put( "credentials", credentials );
+
+ final WebResource putResource = collection.getResource();
+
+ putResource.addFilter( new HTTPBasicAuthFilter(superUserName, superUserPassword) );
+
+
+ putResource.type( MediaType.APPLICATION_JSON_TYPE)
+ .accept( MediaType.APPLICATION_JSON ).put(
+ org.apache.usergrid.rest.test.resource.model.ApiResponse.class, wrapper );
+
+
+ //now try to get a password, it should fail because the hash is no longer correct
+
+ int status = 0;
+
+ // bad access token
+ try {
+ this.app().token().post(new Token("test_1", "test123"));
+ fail("Should have thrown an exception");
+ } catch (UniformInterfaceException uie) {
+ status = uie.getResponse().getStatus();
+ log.info("Error Response Body: " + uie.getResponse().getEntity(String.class));
+ }
+
+ assertEquals(Status.BAD_REQUEST.getStatusCode(), status);
+
+
+ //now put the correct one
+
+
+ credentials.put( "credentials", originalSecret );
+ credentials.put( "secret", originalSecret );
+
+
+ final WebResource putResource2 = collection.getResource();
+
+ putResource2.addFilter( new HTTPBasicAuthFilter( superUserName, superUserPassword ) );
+
+
+ putResource2.type( MediaType.APPLICATION_JSON_TYPE)
+ .accept( MediaType.APPLICATION_JSON ).put(
+ org.apache.usergrid.rest.test.resource.model.ApiResponse.class, wrapper );
+
+
+ //now auth, should be good
+ final Token nextToken = this.app().token().post(new Token("test_1", "test123"));
+
+ assertNotNull( nextToken.getAccessToken() );
+
+ }
}
[30/50] [abbrv] usergrid git commit: Corrections to some comments.
Posted by sn...@apache.org.
Corrections to some comments.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/57a613b5
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/57a613b5
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/57a613b5
Branch: refs/heads/asf-site
Commit: 57a613b58c4cd886c518eaa63e4e8179b3a19cb0
Parents: f8c703c
Author: Dave Johnson <sn...@apache.org>
Authored: Thu Oct 29 17:58:10 2015 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Thu Oct 29 17:58:10 2015 -0400
----------------------------------------------------------------------
stack/scripts/create_test_data.py | 23 ++++++++++++++---------
1 file changed, 14 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/57a613b5/stack/scripts/create_test_data.py
----------------------------------------------------------------------
diff --git a/stack/scripts/create_test_data.py b/stack/scripts/create_test_data.py
index de85da0..ef07254 100644
--- a/stack/scripts/create_test_data.py
+++ b/stack/scripts/create_test_data.py
@@ -27,11 +27,16 @@ import json
# Caller must provide a "slug" string which will be used as a prefix for all names
#
# For example, if the slug is mytest then:
-# Orgs will be named mytest_org0 and mytest_org1
-# Apps will be named mytest_org0_app0 and so on
+#
+# Orgs will be named mytest_org0 and mytest_org1
+# Org admins will be named mytest_org0_admin and mytest_org1_admin (both with password test)
+#
+# Apps will be named mytest_org0_app0, mytest_org0_app1 and so on
# Collections will be named mytest_org0_app0_col0 and so on
-# Entities will be named mytest_org0_app0_col0_entity and so on
-# Org admins will be named mytest_org0_admin and mytest_org1_admin (both with password test)
+# Entities will be named mytest_org0_app0_col0_entity and so on
+#
+# All entities in collection 0 will be connected to entities in collection 1.
+# All entities in collection 1 will be connected to entities in collection 2.
def parse_args():
parser = argparse.ArgumentParser(description="Usergrid Test Data Creation Tool")
@@ -42,7 +47,7 @@ def parse_args():
default="http://localhost:8080")
parser.add_argument("--user",
- help="System Admin Credentials used to authenticate with Usergrid <user:pass>",
+ help="Superuser credentials used to authenticate with Usergrid <user:pass>",
type=str,
required=True)
@@ -56,7 +61,7 @@ def parse_args():
arg_vars = vars(my_args)
creds = arg_vars["user"].split(":")
if len(creds) != 2:
- print("Credentials not properly specified. Must be '-u <user:pass>'. Exiting...")
+ print("Credentials not properly specified. Must be '--user <user:pass>'. Exiting...")
exit_on_error()
else:
arg_vars["user"] = creds[0]
@@ -100,7 +105,7 @@ class Creator:
if ( r.status_code != 200 ):
print "Error logging into organization " + orgName + ": " + r.text
return
-
+
accessToken = r.json()["access_token"]
for appIndex in range(2):
@@ -115,11 +120,11 @@ class Creator:
print " Created app: " + orgName + "/" + appName
appUrl = self.endpoint + "/" + orgName + "/" + appName
- time.sleep(2)
+ time.sleep(2)
for userIndex in range(2):
userName = appName + "_user" + str(userIndex)
- email = userName + "@example.com"
+ email = userName + "@example.com"
url = appUrl + "/users?access_token=" + accessToken
body = json.dumps({"name":userName, "username":userName, "email":email, "password":"test"})
[04/50] [abbrv] usergrid git commit: Moves system endpoints into the
system package
Posted by sn...@apache.org.
Moves system endpoints into the system package
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/7a4f36ab
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/7a4f36ab
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/7a4f36ab
Branch: refs/heads/asf-site
Commit: 7a4f36ab9c0e504133f80f4d0ad8a70d115ee4f4
Parents: 3b43624
Author: Todd Nine <tn...@apigee.com>
Authored: Tue Oct 27 09:46:06 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Tue Oct 27 09:46:06 2015 -0600
----------------------------------------------------------------------
.../usergrid/rest/ApplicationsResource.java | 178 ----------
.../usergrid/rest/ConnectionResource.java | 199 -----------
.../apache/usergrid/rest/DatabaseResource.java | 98 ------
.../org/apache/usergrid/rest/IndexResource.java | 328 ------------------
.../apache/usergrid/rest/MigrateResource.java | 268 ---------------
.../apache/usergrid/rest/SystemResource.java | 108 ------
.../rest/system/ApplicationsResource.java | 178 ++++++++++
.../rest/system/ConnectionResource.java | 202 +++++++++++
.../usergrid/rest/system/DatabaseResource.java | 100 ++++++
.../usergrid/rest/system/IndexResource.java | 331 +++++++++++++++++++
.../usergrid/rest/system/MigrateResource.java | 270 +++++++++++++++
.../usergrid/rest/system/SystemResource.java | 108 ++++++
12 files changed, 1189 insertions(+), 1179 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/7a4f36ab/stack/rest/src/main/java/org/apache/usergrid/rest/ApplicationsResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/ApplicationsResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/ApplicationsResource.java
deleted file mode 100644
index 55cf0f6..0000000
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/ApplicationsResource.java
+++ /dev/null
@@ -1,178 +0,0 @@
-/*
- *
- * * Licensed to the Apache Software Foundation (ASF) under one or more
- * * contributor license agreements. The ASF licenses this file to You
- * * under the Apache License, Version 2.0 (the "License"); you may not
- * * use this file except in compliance with the License.
- * * You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing, software
- * * distributed under the License is distributed on an "AS IS" BASIS,
- * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * * See the License for the specific language governing permissions and
- * * limitations under the License. For additional information regarding
- * * copyright in this work, please see the NOTICE file in the top level
- * * directory of this distribution.
- *
- */
-package org.apache.usergrid.rest;
-
-import com.sun.jersey.api.json.JSONWithPadding;
-import org.apache.usergrid.corepersistence.service.StatusService;
-import org.apache.usergrid.persistence.Entity;
-import org.apache.usergrid.persistence.EntityManager;
-import org.apache.usergrid.persistence.core.util.StringUtils;
-import org.apache.usergrid.persistence.model.util.UUIDGenerator;
-import org.apache.usergrid.rest.security.annotations.RequireSystemAccess;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.context.annotation.Scope;
-import org.springframework.stereotype.Component;
-
-import javax.ws.rs.*;
-import javax.ws.rs.core.Context;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.UriInfo;
-import java.util.HashMap;
-import java.util.LinkedHashMap;
-import java.util.Map;
-import java.util.UUID;
-import java.util.concurrent.atomic.AtomicInteger;
-
-/**
- * Classy class class.
- */
-@Component
-@Scope( "singleton" )
-@Produces( {
- MediaType.APPLICATION_JSON, "application/javascript", "application/x-javascript", "text/ecmascript",
- "application/ecmascript", "text/jscript"
-} )
-public class ApplicationsResource extends AbstractContextResource {
-
- private static final Logger logger = LoggerFactory.getLogger(ApplicationsResource.class);
-
-
- public ApplicationsResource() {
-
- logger.info( "ApplicationsResource initialized" );
- } {
-
- }
-
- @RequireSystemAccess
- @DELETE
- @Path( "{applicationId}" )
- public JSONWithPadding clearApplication( @Context UriInfo ui,
- @PathParam("applicationId") UUID applicationId,
- @QueryParam( "confirmApplicationName" ) String confirmApplicationName,
- @QueryParam( "limit" ) int limit,
- @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
- throws Exception {
-
- if(confirmApplicationName == null){
- throw new IllegalArgumentException("please make add a QueryString for confirmApplicationName");
- }
-
- final UUID jobId = UUIDGenerator.newTimeUUID();
-
- final EntityManager em = emf.getEntityManager(applicationId);
- final String name = em.getApplication().getApplicationName();
- if(!name.toLowerCase().equals(confirmApplicationName.toLowerCase())){
- throw new IllegalArgumentException("confirmApplicationName: " + confirmApplicationName + " does not equal " + name);
- }
- final StatusService statusService = injector.getInstance(StatusService.class);
-
- final ApiResponse response = createApiResponse();
-
- response.setAction( "clear application" );
-
- logger.info("clearing up application");
-
-
- final Thread delete = new Thread() {
-
- @Override
- public void run() {
- final AtomicInteger itemsDeleted = new AtomicInteger(0);
- try {
- management.deleteAllEntities(applicationId, limit)
- .map(id -> itemsDeleted.incrementAndGet())
- .doOnNext(count -> {
- if( count % 100 == 0 ){
- Map<String,Object> map = new LinkedHashMap<>();
- map.put("count",itemsDeleted.intValue());
- final StatusService statusService = injector.getInstance(StatusService.class);
- statusService.setStatus(applicationId, jobId, StatusService.Status.INPROGRESS,map)
- .subscribe();//do not want to throw this exception
- }
- })
- .doOnCompleted(() -> {
- Map<String, Object> map = new LinkedHashMap<>();
- map.put("count", itemsDeleted.intValue());
- final StatusService statusService = injector.getInstance(StatusService.class);
- statusService.setStatus(applicationId, jobId, StatusService.Status.COMPLETE, map)
- .toBlocking().lastOrDefault(null);//want to rethrow this exception
- })
- .toBlocking().lastOrDefault(null);//expecting exception to be caught if job fails
-
- } catch ( Exception e ) {
- Map<String,Object> map = new LinkedHashMap<>();
- map.put("exception",e);
- try {
- statusService.setStatus(applicationId, jobId, StatusService.Status.FAILED, map).toBlocking().lastOrDefault(null);//leave as subscribe if fails retry
- }catch (Exception subE){
- logger.error("failed to update status "+jobId,subE);
- }
- logger.error( "Failed to delete appid:"+applicationId + " jobid:"+jobId+" count:"+itemsDeleted, e );
- }
- }
- };
-
- delete.setName("Delete for app : " + applicationId + " job: " + jobId);
- delete.setDaemon(true);
- delete.start();
-
- try {
- //should throw exception if can't start
- statusService.setStatus(applicationId, jobId, StatusService.Status.STARTED, new LinkedHashMap<>()).toBlocking().lastOrDefault(null);
- }catch (Exception e){
- logger.error("failed to set status for " + jobId, e);
- }
- Map<String,Object> data = new HashMap<>();
- data.put("jobId",jobId);
- data.put("status",StatusService.Status.STARTED);
- response.setData(data);
- response.setSuccess();
- return new JSONWithPadding( response, callback );
- }
-
- @RequireSystemAccess
- @GET
- @Path( "{applicationId}/job/{jobId}" )
- public JSONWithPadding getStatus( @Context UriInfo ui,
- @PathParam("applicationId") UUID applicationId,
- @PathParam("jobId") UUID jobId,
- @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback ) throws Exception{
- final StatusService statusService = injector.getInstance(StatusService.class);
-
- final ApiResponse response = createApiResponse();
-
- response.setAction( "clear application" );
-
- StatusService.JobStatus jobStatus = statusService.getStatus(applicationId, jobId).toBlocking().lastOrDefault(null);
-
- Map<String,Object> data = new HashMap<>();
- data.put("jobId",jobId);
- data.put( "status", jobStatus.getStatus().toString() );
- data.put( "metadata", jobStatus.getData() );
- response.setData(data);
- response.setSuccess();
-
- return new JSONWithPadding( response, callback );
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/7a4f36ab/stack/rest/src/main/java/org/apache/usergrid/rest/ConnectionResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/ConnectionResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/ConnectionResource.java
deleted file mode 100644
index dfa4781..0000000
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/ConnectionResource.java
+++ /dev/null
@@ -1,199 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.usergrid.rest;
-
-
-import java.util.HashMap;
-import java.util.Map;
-import java.util.UUID;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicLong;
-
-import javax.ws.rs.DefaultValue;
-import javax.ws.rs.GET;
-import javax.ws.rs.POST;
-import javax.ws.rs.Path;
-import javax.ws.rs.PathParam;
-import javax.ws.rs.Produces;
-import javax.ws.rs.QueryParam;
-import javax.ws.rs.core.MediaType;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.context.annotation.Scope;
-import org.springframework.stereotype.Component;
-
-import org.apache.usergrid.corepersistence.service.ConnectionService;
-import org.apache.usergrid.corepersistence.service.ConnectionServiceImpl;
-import org.apache.usergrid.corepersistence.service.StatusService;
-import org.apache.usergrid.corepersistence.util.CpNamingUtils;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.index.query.Identifier;
-import org.apache.usergrid.persistence.index.utils.UUIDUtils;
-import org.apache.usergrid.persistence.model.util.UUIDGenerator;
-import org.apache.usergrid.rest.security.annotations.RequireSystemAccess;
-
-import com.google.common.base.Preconditions;
-import com.sun.jersey.api.json.JSONWithPadding;
-
-import rx.Observable;
-import rx.schedulers.Schedulers;
-
-
-/**
- * system/index/otherstuff
- */
-@Component
-@Scope( "singleton" )
-@Produces( {
- MediaType.APPLICATION_JSON, "application/javascript", "application/x-javascript", "text/ecmascript",
- "application/ecmascript", "text/jscript"
-} )
-public class ConnectionResource extends AbstractContextResource {
-
- private static final Logger logger = LoggerFactory.getLogger( ConnectionResource.class );
-
- public ConnectionResource() {
- super();
- }
-
-
- @RequireSystemAccess
- @POST
- @Path( "dedup/" + RootResource.APPLICATION_ID_PATH )
- public JSONWithPadding rebuildIndexesPost( @PathParam( "applicationId" ) String applicationIdStr,
- @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
- throws Exception {
-
-
- logger.info( "Rebuilding all applications" );
-
- final UUID applicationId = UUIDUtils.tryGetUUID( applicationIdStr );
-
- Preconditions.checkNotNull( applicationId, "applicationId must be specified" );
-
- return executeAndCreateResponse( applicationId, callback );
- }
-
-
- @RequireSystemAccess
- @GET
- @Path( "dedup/{jobId: " + Identifier.UUID_REX + "}" )
- public JSONWithPadding rebuildIndexesGet( @PathParam( "jobId" ) String jobId,
- @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
- throws Exception {
- logger.info( "Getting status for index jobs" );
-
- Preconditions.checkNotNull( jobId, "query param jobId must not be null" );
-
-
- final UUID jobUUID = UUIDUtils.tryGetUUID( jobId );
-
- final StatusService.JobStatus
- job = getStatusService().getStatus( CpNamingUtils.MANAGEMENT_APPLICATION_ID, jobUUID ).toBlocking().lastOrDefault(
- null );
-
- Preconditions.checkNotNull( job, "job with id '" + jobId + "' does not exist" );
-
-
- return createResult( job, callback );
- }
-
-
- private ConnectionService getConnectionService() {
- return injector.getInstance( ConnectionServiceImpl.class );
- }
-
-
- private StatusService getStatusService() {
- return injector.getInstance( StatusService.class );
- }
-
-
-
- /**
- * Execute the request and return the response.
- */
- private JSONWithPadding executeAndCreateResponse( final UUID applicationId, final String callback ) {
-
- final Observable<ApplicationScope> applicationScopeObservable =
- Observable.just( CpNamingUtils.getApplicationScope( applicationId ) );
-
- final UUID jobId = UUIDGenerator.newTimeUUID();
-
- final StatusService statusService = getStatusService();
- final ConnectionService connectionService = getConnectionService();
-
- final AtomicLong count = new AtomicLong( 0 );
-
- //start de duping and run in the background
- connectionService.deDupeConnections( applicationScopeObservable ).buffer( 10, TimeUnit.SECONDS, 1000 )
- .doOnNext( buffer -> {
-
-
- final long runningTotal = count.addAndGet( buffer.size() );
-
- final Map<String, Object> status = new HashMap<String, Object>() {{
- put( "countProcessed", runningTotal );
- put( "updatedTimestamp", System.currentTimeMillis() );
- }};
-
- statusService.setStatus( CpNamingUtils.MANAGEMENT_APPLICATION_ID, jobId,
- StatusService.Status.INPROGRESS, status ).toBlocking().lastOrDefault( null );
- } ).doOnSubscribe( () -> {
- statusService.setStatus( CpNamingUtils.MANAGEMENT_APPLICATION_ID, jobId, StatusService.Status.STARTED,
- new HashMap<>() ).toBlocking().lastOrDefault( null );
- } ).doOnCompleted( () -> {
-
- final long runningTotal = count.get();
-
- final Map<String, Object> status = new HashMap<String, Object>() {{
- put( "countProcessed", runningTotal );
- put( "updatedTimestamp", System.currentTimeMillis() );
- }};
-
- statusService
- .setStatus( CpNamingUtils.MANAGEMENT_APPLICATION_ID, jobId, StatusService.Status.COMPLETE, status );
- } ).subscribeOn( Schedulers.newThread() ).subscribe();
-
-
- final StatusService.JobStatus status = new StatusService.JobStatus( jobId, StatusService.Status.STARTED, new HashMap<>( ) );
- return createResult( status, callback );
- }
-
-
- /**
- * Create a response with the specified data.
- * @param jobStatus
- * @param callback
- * @return
- */
- private JSONWithPadding createResult(final StatusService.JobStatus jobStatus, final String callback){
-
- final ApiResponse response = createApiResponse();
-
- response.setAction( "de-dup connections" );
- response.setProperty( "status", jobStatus );
- response.setSuccess();
-
- return new JSONWithPadding( response, callback );
- }
-}
-
-
-
http://git-wip-us.apache.org/repos/asf/usergrid/blob/7a4f36ab/stack/rest/src/main/java/org/apache/usergrid/rest/DatabaseResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/DatabaseResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/DatabaseResource.java
deleted file mode 100644
index a8c5fee..0000000
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/DatabaseResource.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.usergrid.rest;
-
-
-import javax.ws.rs.DefaultValue;
-import javax.ws.rs.PUT;
-import javax.ws.rs.Path;
-import javax.ws.rs.Produces;
-import javax.ws.rs.QueryParam;
-import javax.ws.rs.core.Context;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.UriInfo;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.context.annotation.Scope;
-import org.springframework.stereotype.Component;
-
-import org.apache.usergrid.rest.security.annotations.RequireSystemAccess;
-
-import com.sun.jersey.api.json.JSONWithPadding;
-
-
-@Component
-@Scope( "singleton" )
-@Produces( {
- MediaType.APPLICATION_JSON, "application/javascript", "application/x-javascript", "text/ecmascript",
- "application/ecmascript", "text/jscript"
-} )
-public class DatabaseResource extends AbstractContextResource {
-
- private static final Logger logger = LoggerFactory.getLogger( DatabaseResource.class );
-
-
- public DatabaseResource() {
- logger.info( "DatabaseResource initialized" );
- }
-
-
- @RequireSystemAccess
- @PUT
- @Path( "setup" )
- public JSONWithPadding runDatabaseSetup( @Context UriInfo ui,
- @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
- throws Exception {
-
- ApiResponse response = createApiResponse();
- response.setAction( "cassandra setup" );
-
- logger.info( "Setting up Cassandra" );
-
-
- emf.setup();
-
-
- response.setSuccess();
-
- return new JSONWithPadding( response, callback );
- }
-
-
- @RequireSystemAccess
- @PUT
- @Path( "bootstrap" )
- public JSONWithPadding runSystemSetup( @Context UriInfo ui,
- @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
- throws Exception {
-
- ApiResponse response = createApiResponse();
- response.setAction( "cassandra setup" );
-
- logger.info( "Setting up Cassandra" );
-
-
- emf.boostrap();
- management.setup();
-
- response.setSuccess();
-
- return new JSONWithPadding( response, callback );
- }
-}
-
http://git-wip-us.apache.org/repos/asf/usergrid/blob/7a4f36ab/stack/rest/src/main/java/org/apache/usergrid/rest/IndexResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/IndexResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/IndexResource.java
deleted file mode 100644
index a4a56bc..0000000
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/IndexResource.java
+++ /dev/null
@@ -1,328 +0,0 @@
-/*
- *
- * * Licensed to the Apache Software Foundation (ASF) under one or more
- * * contributor license agreements. The ASF licenses this file to You
- * * under the Apache License, Version 2.0 (the "License"); you may not
- * * use this file except in compliance with the License.
- * * You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing, software
- * * distributed under the License is distributed on an "AS IS" BASIS,
- * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * * See the License for the specific language governing permissions and
- * * limitations under the License. For additional information regarding
- * * copyright in this work, please see the NOTICE file in the top level
- * * directory of this distribution.
- *
- */
-
-package org.apache.usergrid.rest;
-
-
-import com.google.common.base.Preconditions;
-import com.sun.jersey.api.json.JSONWithPadding;
-import org.apache.usergrid.corepersistence.index.ReIndexRequestBuilder;
-import org.apache.usergrid.corepersistence.index.ReIndexRequestBuilderImpl;
-import org.apache.usergrid.corepersistence.index.ReIndexService;
-import org.apache.usergrid.persistence.EntityManager;
-import org.apache.usergrid.persistence.index.utils.ConversionUtils;
-import org.apache.usergrid.persistence.index.utils.UUIDUtils;
-import org.apache.usergrid.rest.security.annotations.RequireSystemAccess;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.context.annotation.Scope;
-import org.springframework.stereotype.Component;
-
-import javax.ws.rs.*;
-import javax.ws.rs.core.Context;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.UriInfo;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.UUID;
-
-
-/**
- * system/index/otherstuff
- */
-@Component
-@Scope( "singleton" )
-@Produces( {
- MediaType.APPLICATION_JSON, "application/javascript", "application/x-javascript", "text/ecmascript",
- "application/ecmascript", "text/jscript"
-} )
-public class IndexResource extends AbstractContextResource {
-
- private static final Logger logger = LoggerFactory.getLogger( IndexResource.class );
- private static final String UPDATED_FIELD = "updated";
-
-
-
- public IndexResource() {
- super();
- }
-
-
- @RequireSystemAccess
- @POST
- @Path( "rebuild" )
- public JSONWithPadding rebuildIndexesPost( @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
- throws Exception {
-
-
- logger.info("Rebuilding all applications");
-
- final ReIndexRequestBuilder request = createRequest();
-
- return executeAndCreateResponse( request, callback );
- }
-
- @RequireSystemAccess
- @GET
- @Path( "rebuild/{jobId}" )
- public JSONWithPadding rebuildIndexesGet(@PathParam( "jobId" ) String jobId, @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
- throws Exception {
- logger.info("Getting status for index jobs");
-
- Preconditions
- .checkNotNull(jobId, "query param jobId must not be null" );
-
-
- ReIndexService.ReIndexStatus status = getReIndexService().getStatus(jobId);
-
- final ApiResponse response = createApiResponse();
-
- response.setAction( "rebuild indexes" );
- response.setProperty( "jobId", status.getJobId() );
- response.setProperty( "status", status.getStatus() );
- response.setProperty( "lastUpdatedEpoch", status.getLastUpdated() );
- response.setProperty( "numberQueued", status.getNumberProcessed() );
- response.setSuccess();
-
- return new JSONWithPadding( response, callback );
- }
-
- @RequireSystemAccess
- @PUT
- @Path( "rebuild" )
- public JSONWithPadding rebuildIndexesPut( final Map<String, Object> payload,
- @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
- throws Exception {
-
-
- logger.info( "Resuming rebuilding all applications" );
- final ReIndexRequestBuilder request = createRequest();
-
- return executeResumeAndCreateResponse( payload, request, callback );
- }
-
-
- @RequireSystemAccess
- @POST
- @Path( "rebuild/" + RootResource.APPLICATION_ID_PATH )
- public JSONWithPadding rebuildIndexesPut( @PathParam( "applicationId" ) String applicationIdStr,
- @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback,
- @QueryParam( "delay" ) @DefaultValue( "10" ) final long delay )
-
- throws Exception {
-
-
- logger.info( "Rebuilding application {}", applicationIdStr );
-
-
- final UUID appId = UUIDUtils.tryExtractUUID( applicationIdStr );
-
- final ReIndexRequestBuilder request = createRequest().withApplicationId( appId );
-
- return executeAndCreateResponse( request, callback );
- }
-
-
- @RequireSystemAccess
- @PUT
- @Path( "rebuild/" + RootResource.APPLICATION_ID_PATH )
- public JSONWithPadding rebuildIndexesPut( final Map<String, Object> payload,
- @PathParam( "applicationId" ) String applicationIdStr,
- @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback,
- @QueryParam( "delay" ) @DefaultValue( "10" ) final long delay )
-
- throws Exception {
-
- logger.info( "Resuming rebuilding application {}", applicationIdStr );
-
- final UUID appId = UUIDUtils.tryExtractUUID( applicationIdStr );
-
- final ReIndexRequestBuilder request = createRequest().withApplicationId( appId );
-
- return executeResumeAndCreateResponse( payload, request, callback );
- }
-
-
- @RequireSystemAccess
- @POST
- @Path( "rebuild/" + RootResource.APPLICATION_ID_PATH + "/{collectionName}" )
- public JSONWithPadding rebuildIndexesPost( @PathParam( "applicationId" ) final String applicationIdStr,
- @PathParam( "collectionName" ) final String collectionName,
- @QueryParam( "reverse" ) @DefaultValue( "false" ) final Boolean reverse,
- @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
- throws Exception {
-
-
- logger.info( "Rebuilding collection {} in application {}", collectionName, applicationIdStr );
-
- final UUID appId = UUIDUtils.tryExtractUUID( applicationIdStr );
-
- final ReIndexRequestBuilder request =
- createRequest().withApplicationId( appId ).withCollection( collectionName );
-
- return executeAndCreateResponse( request, callback );
- }
-
-
- @RequireSystemAccess
- @PUT
- @Path( "rebuild/" + RootResource.APPLICATION_ID_PATH + "/{collectionName}" )
- public JSONWithPadding rebuildIndexesPut( final Map<String, Object> payload,
- @PathParam( "applicationId" ) final String applicationIdStr,
- @PathParam( "collectionName" ) final String collectionName,
- @QueryParam( "reverse" ) @DefaultValue( "false" ) final Boolean reverse,
- @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
- throws Exception {
-
- logger.info( "Resuming rebuilding collection {} in application {}", collectionName, applicationIdStr );
-
- final UUID appId = UUIDUtils.tryExtractUUID( applicationIdStr );
-
- final ReIndexRequestBuilder request =
- createRequest().withApplicationId( appId ).withCollection( collectionName );
-
- return executeResumeAndCreateResponse( payload, request, callback );
- }
-
-
- @RequireSystemAccess
- @POST
- @Path( "rebuild/management" )
- public JSONWithPadding rebuildInternalIndexesPost(
- @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback ) throws Exception {
-
-
- final UUID managementAppId = emf.getManagementAppId();
-
- logger.info( "Rebuilding management application with id {} ", managementAppId );
- final ReIndexRequestBuilder request = createRequest().withApplicationId( managementAppId );
-
- return executeAndCreateResponse( request, callback );
- }
-
-
- @RequireSystemAccess
- @PUT
- @Path( "rebuild/management" )
- public JSONWithPadding rebuildInternalIndexesPut( final Map<String, Object> payload,
- @QueryParam( "callback" ) @DefaultValue( "callback" )
- String callback ) throws Exception {
-
-
- final UUID managementAppId = emf.getManagementAppId();
-
- logger.info( "Resuming rebuilding management application with id {} ", managementAppId );
- final ReIndexRequestBuilder request = createRequest().withApplicationId( managementAppId );
-
- return executeResumeAndCreateResponse( payload, request, callback );
- }
-
-
- @RequireSystemAccess
- @POST
- @Path(RootResource.APPLICATION_ID_PATH)
- public JSONWithPadding addIndex( @Context UriInfo ui,
- @PathParam( "applicationId" ) final String applicationIdStr,
- Map<String, Object> config,
- @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
- throws Exception {
-
- Preconditions
- .checkNotNull( config, "Payload for config is null, please pass {replicas:int, shards:int} in body" );
-
- ApiResponse response = createApiResponse();
-
- if ( !config.containsKey( "replicas" ) || !config.containsKey( "shards" ) ||
- !( config.get( "replicas" ) instanceof Integer ) || !( config.get( "shards" ) instanceof Integer ) ) {
- throw new IllegalArgumentException( "body must contains 'replicas' of type int and 'shards' of type int" );
- }
-
- if ( !config.containsKey( "indexSuffix" ) ) {
- throw new IllegalArgumentException( "Please add an indexSuffix to your post" );
- }
- final UUID appId = UUIDUtils.tryExtractUUID( applicationIdStr );
-
- if(appId == null){
- throw new IllegalArgumentException("app id was not parsed");
- }
-
- EntityManager em = emf.getEntityManager(appId);
- em.addIndex(config.get("indexSuffix").toString(), (int) config.get("shards"),
- (int) config.get("replicas"), (String) config.get("writeConsistency"));
- response.setAction( "Add index to alias" );
-
- return new JSONWithPadding( response, callback );
- }
-
-
- private ReIndexService getReIndexService() {
- return injector.getInstance( ReIndexService.class );
- }
-
-
- private ReIndexRequestBuilder createRequest() {
- //TODO: wire this up through spring, and in the future guice.
- return new ReIndexRequestBuilderImpl();
- }
-
-
- private JSONWithPadding executeResumeAndCreateResponse( final Map<String, Object> payload,
- final ReIndexRequestBuilder request,
- final String callback ) {
-
- Map<String,Object> newPayload = payload;
- if(newPayload == null || !payload.containsKey( UPDATED_FIELD )){
- newPayload = new HashMap<>(1);
- newPayload.put(UPDATED_FIELD,0);
- }
-
- Preconditions.checkArgument(newPayload.get(UPDATED_FIELD) instanceof Number,
- "You must specified the field \"updated\" in the payload and it must be a timestamp" );
-
- //add our updated timestamp to the request
- if ( newPayload.containsKey( UPDATED_FIELD ) ) {
- final long timestamp = ConversionUtils.getLong(newPayload.get(UPDATED_FIELD));
- request.withStartTimestamp( timestamp );
- }
-
- return executeAndCreateResponse( request, callback );
- }
-
-
- /**
- * Execute the request and return the response.
- */
- private JSONWithPadding executeAndCreateResponse( final ReIndexRequestBuilder request, final String callback ) {
-
-
- final ReIndexService.ReIndexStatus status = getReIndexService().rebuildIndex( request );
-
- final ApiResponse response = createApiResponse();
-
- response.setAction( "rebuild indexes" );
- response.setProperty( "jobId", status.getJobId() );
- response.setProperty( "status", status.getStatus() );
- response.setProperty( "lastUpdatedEpoch", status.getLastUpdated() );
- response.setProperty( "numberQueued", status.getNumberProcessed() );
- response.setSuccess();
-
- return new JSONWithPadding( response, callback );
- }
-}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/7a4f36ab/stack/rest/src/main/java/org/apache/usergrid/rest/MigrateResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/MigrateResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/MigrateResource.java
deleted file mode 100644
index 7410927..0000000
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/MigrateResource.java
+++ /dev/null
@@ -1,268 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.usergrid.rest;
-
-
-import java.util.Map;
-import java.util.Set;
-
-import javax.ws.rs.*;
-import javax.ws.rs.core.Context;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.UriInfo;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.context.annotation.Scope;
-import org.springframework.stereotype.Component;
-
-import org.apache.usergrid.persistence.core.migration.data.DataMigrationManager;
-import org.apache.usergrid.persistence.core.migration.schema.MigrationManager;
-import org.apache.usergrid.rest.security.annotations.RequireSystemAccess;
-
-import com.fasterxml.jackson.databind.node.JsonNodeFactory;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import com.google.common.base.Preconditions;
-import com.google.inject.Injector;
-import com.sun.jersey.api.json.JSONWithPadding;
-
-
-@Component
-@Scope( "singleton" )
-@Produces( {
- MediaType.APPLICATION_JSON,
- "application/javascript",
- "application/x-javascript",
- "text/ecmascript",
- "application/ecmascript",
- "text/jscript"
-} )
-public class MigrateResource extends AbstractContextResource {
-
- private static final Logger logger = LoggerFactory.getLogger( MigrateResource.class );
-
- public MigrateResource() {
- logger.info( "SystemResource initialized" );
- }
-
- @Autowired
- private Injector guiceInjector;
-
- @RequireSystemAccess
- @PUT
- @Path( "run" )
- public JSONWithPadding migrateData( @Context UriInfo ui,
- @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
- throws Exception {
-
- ApiResponse response = createApiResponse();
- response.setAction( "Migrate Data" );
- //TODO make this use the task scheduler
-
-
- final Thread migrate = new Thread() {
-
- @Override
- public void run() {
-
- logger.info( "Migrating Schema" );
-
- try {
- getMigrationManager().migrate();
- }
- catch ( Exception e ) {
- logger.error( "Unable to migrate data", e );
- }
-
- logger.info( "Migrating Data" );
-
- try {
- getDataMigrationManager().migrate();
- }
- catch ( Exception e ) {
- logger.error( "Unable to migrate data", e );
- }
- }
- };
-
- migrate.setName( "Index migrate data formats" );
- migrate.setDaemon( true );
- migrate.start();
-
- response.setSuccess();
-
- return new JSONWithPadding( response, callback );
- }
-
- @RequireSystemAccess
- @PUT
- @Path( "run/{pluginName}" )
- public JSONWithPadding migrateData(@PathParam("pluginName") String pluginName , @Context UriInfo ui,
- @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
- throws Exception {
-
- if(!getDataMigrationManager().pluginExists(pluginName)){
- throw new IllegalArgumentException("Plugin doesn't exits name:"+pluginName);
- }
-
- ApiResponse response = createApiResponse();
- response.setAction( "Migrate Data: "+ pluginName );
- //TODO make this use the task scheduler
-
-
-
-
- final Thread migrate = new Thread() {
-
- @Override
- public void run() {
-
- logger.info( "Migrating Data for plugin: " + pluginName );
-
- try {
- getDataMigrationManager().migrate(pluginName);
- }
- catch ( Exception e ) {
- logger.error( "Unable to migrate data for plugin: " + pluginName, e );
- }
- }
- };
-
- migrate.setName( "Index migrate data formats: "+pluginName );
- migrate.setDaemon( true );
- migrate.start();
-
- response.setSuccess();
-
- return new JSONWithPadding( response, callback );
- }
-
- @RequireSystemAccess
- @PUT
- @Path( "set" )
- public JSONWithPadding setMigrationVersion(
- @Context UriInfo ui, Map<String, Object> json,
- @QueryParam( "callback" ) @DefaultValue( "" ) String callback )
- throws Exception {
-
- logger.debug( "setMigrationVersion" );
-
- Preconditions.checkNotNull( json, "You must provide a json body" );
- Preconditions.checkArgument( json.keySet().size() > 0, "You must specify at least one module and version" );
-
- ApiResponse response = createApiResponse();
- response.setAction("Set Migration Versions");
-
- ObjectNode node = JsonNodeFactory.instance.objectNode();
-
- final DataMigrationManager dataMigrationManager = getDataMigrationManager();
- final Set<String> plugins = dataMigrationManager.getPluginNames();
-
- /**
- * Set the migration version for the plugins specified
- */
- for ( final String key : json.keySet() ) {
-
- int version = ( int ) json.get( key );
-
- dataMigrationManager.resetToVersion(key, version);
- }
-
-
- /**
- * Echo back a response of the current versions for all plugins
- */
- for(final String pluginName: plugins){
- node.put(pluginName, dataMigrationManager.getCurrentVersion(pluginName));
- }
-
-
- response.setData( node );
- response.setSuccess();
-
- return new JSONWithPadding( response, callback );
- }
-
-
- @RequireSystemAccess
- @GET
- @Path( "status" )
- public JSONWithPadding migrateStatus(
- @Context UriInfo ui,
- @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
- throws Exception {
-
- ApiResponse response = createApiResponse();
- response.setAction( "Migrate Schema indexes" );
-
- ObjectNode node = JsonNodeFactory.instance.objectNode();
-
-
-
- final DataMigrationManager dataMigrationManager = getDataMigrationManager();
-
- final Set<String> plugins = dataMigrationManager.getPluginNames();
-
- for(final String pluginName: plugins){
- node.put( pluginName, dataMigrationManager.getCurrentVersion( pluginName ) );
- }
-
- response.setData( node );
-
- response.setSuccess();
-
- return new JSONWithPadding( response, callback );
- }
-
-
- @RequireSystemAccess
- @GET
- @Path( "count" )
- public JSONWithPadding migrateCount(
- @Context UriInfo ui,
- @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
- throws Exception {
-
- ApiResponse response = createApiResponse();
- response.setAction( "Current entity count in system" );
-
- response.setProperty( "count", emf.performEntityCount() );
-
- response.setSuccess();
-
- return new JSONWithPadding( response, callback );
- }
-
-
- /**
- * Get the schema migration manager
- */
- private MigrationManager getMigrationManager() {
- return guiceInjector.getInstance( MigrationManager.class );
- }
-
- /**
- * Get the Data migration manager
- */
- private DataMigrationManager getDataMigrationManager() {
- return guiceInjector.getInstance( DataMigrationManager.class );
- }
-}
-
http://git-wip-us.apache.org/repos/asf/usergrid/blob/7a4f36ab/stack/rest/src/main/java/org/apache/usergrid/rest/SystemResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/SystemResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/SystemResource.java
deleted file mode 100644
index aaee596..0000000
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/SystemResource.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.usergrid.rest;
-
-
-import javax.ws.rs.*;
-import javax.ws.rs.core.Context;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.UriInfo;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.context.annotation.Scope;
-import org.springframework.stereotype.Component;
-
-import org.apache.usergrid.rest.security.annotations.RequireSystemAccess;
-
-import com.sun.jersey.api.json.JSONWithPadding;
-
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.UUID;
-import java.util.concurrent.atomic.AtomicInteger;
-
-
-@Path( "/system" )
-@Component
-@Scope( "singleton" )
-@Produces( {
- MediaType.APPLICATION_JSON, "application/javascript", "application/x-javascript", "text/ecmascript",
- "application/ecmascript", "text/jscript"
-} )
-public class SystemResource extends AbstractContextResource {
-
- private static final Logger logger = LoggerFactory.getLogger( SystemResource.class );
-
-
- public SystemResource() {
- logger.info( "SystemResource initialized" );
- }
-
-
- @RequireSystemAccess
- @GET
- @Path( "superuser/setup" )
- public JSONWithPadding getSetupSuperuser( @Context UriInfo ui,
- @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
- throws Exception {
-
- ApiResponse response = createApiResponse();
- response.setAction( "superuser setup" );
-
- logger.info( "Setting up Superuser" );
-
- try {
- management.provisionSuperuser();
- }
- catch ( Exception e ) {
- logger.error( "Unable to complete superuser setup", e );
- }
-
- response.setSuccess();
-
- return new JSONWithPadding( response, callback );
- }
-
-
-
- @Path( "migrate" )
- public MigrateResource migrate() {
- return getSubResource( MigrateResource.class );
- }
-
-
- @Path( "index" )
- public IndexResource index() { return getSubResource( IndexResource.class ); }
-
-
- @Path( "database" )
- public DatabaseResource database() {
- return getSubResource( DatabaseResource.class );
- }
-
- @Path( "applications" )
- public ApplicationsResource applications() {
- return getSubResource( ApplicationsResource.class );
- }
-
-
- @Path( "connection" )
- public ConnectionResource connection() { return getSubResource( ConnectionResource.class ); }
-
-}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/7a4f36ab/stack/rest/src/main/java/org/apache/usergrid/rest/system/ApplicationsResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/system/ApplicationsResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/system/ApplicationsResource.java
new file mode 100644
index 0000000..82088f9
--- /dev/null
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/system/ApplicationsResource.java
@@ -0,0 +1,178 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. The ASF licenses this file to You
+ * * under the Apache License, Version 2.0 (the "License"); you may not
+ * * use this file except in compliance with the License.
+ * * You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License. For additional information regarding
+ * * copyright in this work, please see the NOTICE file in the top level
+ * * directory of this distribution.
+ *
+ */
+package org.apache.usergrid.rest.system;
+
+import com.sun.jersey.api.json.JSONWithPadding;
+import org.apache.usergrid.corepersistence.service.StatusService;
+import org.apache.usergrid.persistence.EntityManager;
+import org.apache.usergrid.persistence.model.util.UUIDGenerator;
+import org.apache.usergrid.rest.AbstractContextResource;
+import org.apache.usergrid.rest.ApiResponse;
+import org.apache.usergrid.rest.security.annotations.RequireSystemAccess;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.context.annotation.Scope;
+import org.springframework.stereotype.Component;
+
+import javax.ws.rs.*;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.UriInfo;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * Classy class class.
+ */
+@Component
+@Scope( "singleton" )
+@Produces( {
+ MediaType.APPLICATION_JSON, "application/javascript", "application/x-javascript", "text/ecmascript",
+ "application/ecmascript", "text/jscript"
+} )
+public class ApplicationsResource extends AbstractContextResource {
+
+ private static final Logger logger = LoggerFactory.getLogger(ApplicationsResource.class);
+
+
+ public ApplicationsResource() {
+
+ logger.info( "ApplicationsResource initialized" );
+ } {
+
+ }
+
+ @RequireSystemAccess
+ @DELETE
+ @Path( "{applicationId}" )
+ public JSONWithPadding clearApplication( @Context UriInfo ui,
+ @PathParam("applicationId") UUID applicationId,
+ @QueryParam( "confirmApplicationName" ) String confirmApplicationName,
+ @QueryParam( "limit" ) int limit,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+ throws Exception {
+
+ if(confirmApplicationName == null){
+ throw new IllegalArgumentException("please make add a QueryString for confirmApplicationName");
+ }
+
+ final UUID jobId = UUIDGenerator.newTimeUUID();
+
+ final EntityManager em = emf.getEntityManager(applicationId);
+ final String name = em.getApplication().getApplicationName();
+ if(!name.toLowerCase().equals(confirmApplicationName.toLowerCase())){
+ throw new IllegalArgumentException("confirmApplicationName: " + confirmApplicationName + " does not equal " + name);
+ }
+ final StatusService statusService = injector.getInstance(StatusService.class);
+
+ final ApiResponse response = createApiResponse();
+
+ response.setAction( "clear application" );
+
+ logger.info("clearing up application");
+
+
+ final Thread delete = new Thread() {
+
+ @Override
+ public void run() {
+ final AtomicInteger itemsDeleted = new AtomicInteger(0);
+ try {
+ management.deleteAllEntities(applicationId, limit)
+ .map(id -> itemsDeleted.incrementAndGet())
+ .doOnNext(count -> {
+ if( count % 100 == 0 ){
+ Map<String,Object> map = new LinkedHashMap<>();
+ map.put("count",itemsDeleted.intValue());
+ final StatusService statusService = injector.getInstance(StatusService.class);
+ statusService.setStatus(applicationId, jobId, StatusService.Status.INPROGRESS,map)
+ .subscribe();//do not want to throw this exception
+ }
+ })
+ .doOnCompleted(() -> {
+ Map<String, Object> map = new LinkedHashMap<>();
+ map.put("count", itemsDeleted.intValue());
+ final StatusService statusService = injector.getInstance(StatusService.class);
+ statusService.setStatus(applicationId, jobId, StatusService.Status.COMPLETE, map)
+ .toBlocking().lastOrDefault(null);//want to rethrow this exception
+ })
+ .toBlocking().lastOrDefault(null);//expecting exception to be caught if job fails
+
+ } catch ( Exception e ) {
+ Map<String,Object> map = new LinkedHashMap<>();
+ map.put("exception",e);
+ try {
+ statusService.setStatus(applicationId, jobId, StatusService.Status.FAILED, map).toBlocking().lastOrDefault(null);//leave as subscribe if fails retry
+ }catch (Exception subE){
+ logger.error("failed to update status "+jobId,subE);
+ }
+ logger.error( "Failed to delete appid:"+applicationId + " jobid:"+jobId+" count:"+itemsDeleted, e );
+ }
+ }
+ };
+
+ delete.setName("Delete for app : " + applicationId + " job: " + jobId);
+ delete.setDaemon(true);
+ delete.start();
+
+ try {
+ //should throw exception if can't start
+ statusService.setStatus(applicationId, jobId, StatusService.Status.STARTED, new LinkedHashMap<>()).toBlocking().lastOrDefault(null);
+ }catch (Exception e){
+ logger.error("failed to set status for " + jobId, e);
+ }
+ Map<String,Object> data = new HashMap<>();
+ data.put("jobId",jobId);
+ data.put("status",StatusService.Status.STARTED);
+ response.setData(data);
+ response.setSuccess();
+ return new JSONWithPadding( response, callback );
+ }
+
+ @RequireSystemAccess
+ @GET
+ @Path( "{applicationId}/job/{jobId}" )
+ public JSONWithPadding getStatus( @Context UriInfo ui,
+ @PathParam("applicationId") UUID applicationId,
+ @PathParam("jobId") UUID jobId,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback ) throws Exception{
+ final StatusService statusService = injector.getInstance(StatusService.class);
+
+ final ApiResponse response = createApiResponse();
+
+ response.setAction( "clear application" );
+
+ StatusService.JobStatus jobStatus = statusService.getStatus(applicationId, jobId).toBlocking().lastOrDefault(null);
+
+ Map<String,Object> data = new HashMap<>();
+ data.put("jobId",jobId);
+ data.put( "status", jobStatus.getStatus().toString() );
+ data.put( "metadata", jobStatus.getData() );
+ response.setData(data);
+ response.setSuccess();
+
+ return new JSONWithPadding( response, callback );
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/7a4f36ab/stack/rest/src/main/java/org/apache/usergrid/rest/system/ConnectionResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/system/ConnectionResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/system/ConnectionResource.java
new file mode 100644
index 0000000..6e683ed
--- /dev/null
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/system/ConnectionResource.java
@@ -0,0 +1,202 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.usergrid.rest.system;
+
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+
+import javax.ws.rs.DefaultValue;
+import javax.ws.rs.GET;
+import javax.ws.rs.POST;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.core.MediaType;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.context.annotation.Scope;
+import org.springframework.stereotype.Component;
+
+import org.apache.usergrid.corepersistence.service.ConnectionService;
+import org.apache.usergrid.corepersistence.service.ConnectionServiceImpl;
+import org.apache.usergrid.corepersistence.service.StatusService;
+import org.apache.usergrid.corepersistence.util.CpNamingUtils;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.index.query.Identifier;
+import org.apache.usergrid.persistence.index.utils.UUIDUtils;
+import org.apache.usergrid.persistence.model.util.UUIDGenerator;
+import org.apache.usergrid.rest.AbstractContextResource;
+import org.apache.usergrid.rest.ApiResponse;
+import org.apache.usergrid.rest.RootResource;
+import org.apache.usergrid.rest.security.annotations.RequireSystemAccess;
+
+import com.google.common.base.Preconditions;
+import com.sun.jersey.api.json.JSONWithPadding;
+
+import rx.Observable;
+import rx.schedulers.Schedulers;
+
+
+/**
+ * system/index/otherstuff
+ */
+@Component
+@Scope( "singleton" )
+@Produces( {
+ MediaType.APPLICATION_JSON, "application/javascript", "application/x-javascript", "text/ecmascript",
+ "application/ecmascript", "text/jscript"
+} )
+public class ConnectionResource extends AbstractContextResource {
+
+ private static final Logger logger = LoggerFactory.getLogger( ConnectionResource.class );
+
+ public ConnectionResource() {
+ super();
+ }
+
+
+ @RequireSystemAccess
+ @POST
+ @Path( "dedup/" + RootResource.APPLICATION_ID_PATH )
+ public JSONWithPadding rebuildIndexesPost( @PathParam( "applicationId" ) String applicationIdStr,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+ throws Exception {
+
+
+ logger.info( "Rebuilding all applications" );
+
+ final UUID applicationId = UUIDUtils.tryGetUUID( applicationIdStr );
+
+ Preconditions.checkNotNull( applicationId, "applicationId must be specified" );
+
+ return executeAndCreateResponse( applicationId, callback );
+ }
+
+
+ @RequireSystemAccess
+ @GET
+ @Path( "dedup/{jobId: " + Identifier.UUID_REX + "}" )
+ public JSONWithPadding rebuildIndexesGet( @PathParam( "jobId" ) String jobId,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+ throws Exception {
+ logger.info( "Getting status for index jobs" );
+
+ Preconditions.checkNotNull( jobId, "query param jobId must not be null" );
+
+
+ final UUID jobUUID = UUIDUtils.tryGetUUID( jobId );
+
+ final StatusService.JobStatus
+ job = getStatusService().getStatus( CpNamingUtils.MANAGEMENT_APPLICATION_ID, jobUUID ).toBlocking().lastOrDefault(
+ null );
+
+ Preconditions.checkNotNull( job, "job with id '" + jobId + "' does not exist" );
+
+
+ return createResult( job, callback );
+ }
+
+
+ private ConnectionService getConnectionService() {
+ return injector.getInstance( ConnectionServiceImpl.class );
+ }
+
+
+ private StatusService getStatusService() {
+ return injector.getInstance( StatusService.class );
+ }
+
+
+
+ /**
+ * Execute the request and return the response.
+ */
+ private JSONWithPadding executeAndCreateResponse( final UUID applicationId, final String callback ) {
+
+ final Observable<ApplicationScope> applicationScopeObservable =
+ Observable.just( CpNamingUtils.getApplicationScope( applicationId ) );
+
+ final UUID jobId = UUIDGenerator.newTimeUUID();
+
+ final StatusService statusService = getStatusService();
+ final ConnectionService connectionService = getConnectionService();
+
+ final AtomicLong count = new AtomicLong( 0 );
+
+ //start de duping and run in the background
+ connectionService.deDupeConnections( applicationScopeObservable ).buffer( 10, TimeUnit.SECONDS, 1000 )
+ .doOnNext( buffer -> {
+
+
+ final long runningTotal = count.addAndGet( buffer.size() );
+
+ final Map<String, Object> status = new HashMap<String, Object>() {{
+ put( "countProcessed", runningTotal );
+ put( "updatedTimestamp", System.currentTimeMillis() );
+ }};
+
+ statusService.setStatus( CpNamingUtils.MANAGEMENT_APPLICATION_ID, jobId,
+ StatusService.Status.INPROGRESS, status ).toBlocking().lastOrDefault( null );
+ } ).doOnSubscribe( () -> {
+ statusService.setStatus( CpNamingUtils.MANAGEMENT_APPLICATION_ID, jobId, StatusService.Status.STARTED,
+ new HashMap<>() ).toBlocking().lastOrDefault( null );
+ } ).doOnCompleted( () -> {
+
+ final long runningTotal = count.get();
+
+ final Map<String, Object> status = new HashMap<String, Object>() {{
+ put( "countProcessed", runningTotal );
+ put( "updatedTimestamp", System.currentTimeMillis() );
+ }};
+
+ statusService
+ .setStatus( CpNamingUtils.MANAGEMENT_APPLICATION_ID, jobId, StatusService.Status.COMPLETE, status );
+ } ).subscribeOn( Schedulers.newThread() ).subscribe();
+
+
+ final StatusService.JobStatus status = new StatusService.JobStatus( jobId, StatusService.Status.STARTED, new HashMap<>( ) );
+ return createResult( status, callback );
+ }
+
+
+ /**
+ * Create a response with the specified data.
+ * @param jobStatus
+ * @param callback
+ * @return
+ */
+ private JSONWithPadding createResult(final StatusService.JobStatus jobStatus, final String callback){
+
+ final ApiResponse response = createApiResponse();
+
+ response.setAction( "de-dup connections" );
+ response.setProperty( "status", jobStatus );
+ response.setSuccess();
+
+ return new JSONWithPadding( response, callback );
+ }
+}
+
+
+
http://git-wip-us.apache.org/repos/asf/usergrid/blob/7a4f36ab/stack/rest/src/main/java/org/apache/usergrid/rest/system/DatabaseResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/system/DatabaseResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/system/DatabaseResource.java
new file mode 100644
index 0000000..42a63ca
--- /dev/null
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/system/DatabaseResource.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.rest.system;
+
+
+import javax.ws.rs.DefaultValue;
+import javax.ws.rs.PUT;
+import javax.ws.rs.Path;
+import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.UriInfo;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.context.annotation.Scope;
+import org.springframework.stereotype.Component;
+
+import org.apache.usergrid.rest.AbstractContextResource;
+import org.apache.usergrid.rest.ApiResponse;
+import org.apache.usergrid.rest.security.annotations.RequireSystemAccess;
+
+import com.sun.jersey.api.json.JSONWithPadding;
+
+
+@Component
+@Scope( "singleton" )
+@Produces( {
+ MediaType.APPLICATION_JSON, "application/javascript", "application/x-javascript", "text/ecmascript",
+ "application/ecmascript", "text/jscript"
+} )
+public class DatabaseResource extends AbstractContextResource {
+
+ private static final Logger logger = LoggerFactory.getLogger( DatabaseResource.class );
+
+
+ public DatabaseResource() {
+ logger.info( "DatabaseResource initialized" );
+ }
+
+
+ @RequireSystemAccess
+ @PUT
+ @Path( "setup" )
+ public JSONWithPadding runDatabaseSetup( @Context UriInfo ui,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+ throws Exception {
+
+ ApiResponse response = createApiResponse();
+ response.setAction( "cassandra setup" );
+
+ logger.info( "Setting up Cassandra" );
+
+
+ emf.setup();
+
+
+ response.setSuccess();
+
+ return new JSONWithPadding( response, callback );
+ }
+
+
+ @RequireSystemAccess
+ @PUT
+ @Path( "bootstrap" )
+ public JSONWithPadding runSystemSetup( @Context UriInfo ui,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+ throws Exception {
+
+ ApiResponse response = createApiResponse();
+ response.setAction( "cassandra setup" );
+
+ logger.info( "Setting up Cassandra" );
+
+
+ emf.boostrap();
+ management.setup();
+
+ response.setSuccess();
+
+ return new JSONWithPadding( response, callback );
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/usergrid/blob/7a4f36ab/stack/rest/src/main/java/org/apache/usergrid/rest/system/IndexResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/system/IndexResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/system/IndexResource.java
new file mode 100644
index 0000000..8e2946c
--- /dev/null
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/system/IndexResource.java
@@ -0,0 +1,331 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. The ASF licenses this file to You
+ * * under the Apache License, Version 2.0 (the "License"); you may not
+ * * use this file except in compliance with the License.
+ * * You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License. For additional information regarding
+ * * copyright in this work, please see the NOTICE file in the top level
+ * * directory of this distribution.
+ *
+ */
+
+package org.apache.usergrid.rest.system;
+
+
+import com.google.common.base.Preconditions;
+import com.sun.jersey.api.json.JSONWithPadding;
+import org.apache.usergrid.corepersistence.index.ReIndexRequestBuilder;
+import org.apache.usergrid.corepersistence.index.ReIndexRequestBuilderImpl;
+import org.apache.usergrid.corepersistence.index.ReIndexService;
+import org.apache.usergrid.persistence.EntityManager;
+import org.apache.usergrid.persistence.index.utils.ConversionUtils;
+import org.apache.usergrid.persistence.index.utils.UUIDUtils;
+import org.apache.usergrid.rest.AbstractContextResource;
+import org.apache.usergrid.rest.ApiResponse;
+import org.apache.usergrid.rest.RootResource;
+import org.apache.usergrid.rest.security.annotations.RequireSystemAccess;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.context.annotation.Scope;
+import org.springframework.stereotype.Component;
+
+import javax.ws.rs.*;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.UriInfo;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.UUID;
+
+
+/**
+ * system/index/otherstuff
+ */
+@Component
+@Scope( "singleton" )
+@Produces( {
+ MediaType.APPLICATION_JSON, "application/javascript", "application/x-javascript", "text/ecmascript",
+ "application/ecmascript", "text/jscript"
+} )
+public class IndexResource extends AbstractContextResource {
+
+ private static final Logger logger = LoggerFactory.getLogger( IndexResource.class );
+ private static final String UPDATED_FIELD = "updated";
+
+
+
+ public IndexResource() {
+ super();
+ }
+
+
+ @RequireSystemAccess
+ @POST
+ @Path( "rebuild" )
+ public JSONWithPadding rebuildIndexesPost( @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+ throws Exception {
+
+
+ logger.info("Rebuilding all applications");
+
+ final ReIndexRequestBuilder request = createRequest();
+
+ return executeAndCreateResponse( request, callback );
+ }
+
+ @RequireSystemAccess
+ @GET
+ @Path( "rebuild/{jobId}" )
+ public JSONWithPadding rebuildIndexesGet(@PathParam( "jobId" ) String jobId, @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+ throws Exception {
+ logger.info("Getting status for index jobs");
+
+ Preconditions
+ .checkNotNull(jobId, "query param jobId must not be null" );
+
+
+ ReIndexService.ReIndexStatus status = getReIndexService().getStatus(jobId);
+
+ final ApiResponse response = createApiResponse();
+
+ response.setAction( "rebuild indexes" );
+ response.setProperty( "jobId", status.getJobId() );
+ response.setProperty( "status", status.getStatus() );
+ response.setProperty( "lastUpdatedEpoch", status.getLastUpdated() );
+ response.setProperty( "numberQueued", status.getNumberProcessed() );
+ response.setSuccess();
+
+ return new JSONWithPadding( response, callback );
+ }
+
+ @RequireSystemAccess
+ @PUT
+ @Path( "rebuild" )
+ public JSONWithPadding rebuildIndexesPut( final Map<String, Object> payload,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+ throws Exception {
+
+
+ logger.info( "Resuming rebuilding all applications" );
+ final ReIndexRequestBuilder request = createRequest();
+
+ return executeResumeAndCreateResponse( payload, request, callback );
+ }
+
+
+ @RequireSystemAccess
+ @POST
+ @Path( "rebuild/" + RootResource.APPLICATION_ID_PATH )
+ public JSONWithPadding rebuildIndexesPut( @PathParam( "applicationId" ) String applicationIdStr,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback,
+ @QueryParam( "delay" ) @DefaultValue( "10" ) final long delay )
+
+ throws Exception {
+
+
+ logger.info( "Rebuilding application {}", applicationIdStr );
+
+
+ final UUID appId = UUIDUtils.tryExtractUUID( applicationIdStr );
+
+ final ReIndexRequestBuilder request = createRequest().withApplicationId( appId );
+
+ return executeAndCreateResponse( request, callback );
+ }
+
+
+ @RequireSystemAccess
+ @PUT
+ @Path( "rebuild/" + RootResource.APPLICATION_ID_PATH )
+ public JSONWithPadding rebuildIndexesPut( final Map<String, Object> payload,
+ @PathParam( "applicationId" ) String applicationIdStr,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback,
+ @QueryParam( "delay" ) @DefaultValue( "10" ) final long delay )
+
+ throws Exception {
+
+ logger.info( "Resuming rebuilding application {}", applicationIdStr );
+
+ final UUID appId = UUIDUtils.tryExtractUUID( applicationIdStr );
+
+ final ReIndexRequestBuilder request = createRequest().withApplicationId( appId );
+
+ return executeResumeAndCreateResponse( payload, request, callback );
+ }
+
+
+ @RequireSystemAccess
+ @POST
+ @Path( "rebuild/" + RootResource.APPLICATION_ID_PATH + "/{collectionName}" )
+ public JSONWithPadding rebuildIndexesPost( @PathParam( "applicationId" ) final String applicationIdStr,
+ @PathParam( "collectionName" ) final String collectionName,
+ @QueryParam( "reverse" ) @DefaultValue( "false" ) final Boolean reverse,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+ throws Exception {
+
+
+ logger.info( "Rebuilding collection {} in application {}", collectionName, applicationIdStr );
+
+ final UUID appId = UUIDUtils.tryExtractUUID( applicationIdStr );
+
+ final ReIndexRequestBuilder request =
+ createRequest().withApplicationId( appId ).withCollection( collectionName );
+
+ return executeAndCreateResponse( request, callback );
+ }
+
+
+ @RequireSystemAccess
+ @PUT
+ @Path( "rebuild/" + RootResource.APPLICATION_ID_PATH + "/{collectionName}" )
+ public JSONWithPadding rebuildIndexesPut( final Map<String, Object> payload,
+ @PathParam( "applicationId" ) final String applicationIdStr,
+ @PathParam( "collectionName" ) final String collectionName,
+ @QueryParam( "reverse" ) @DefaultValue( "false" ) final Boolean reverse,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+ throws Exception {
+
+ logger.info( "Resuming rebuilding collection {} in application {}", collectionName, applicationIdStr );
+
+ final UUID appId = UUIDUtils.tryExtractUUID( applicationIdStr );
+
+ final ReIndexRequestBuilder request =
+ createRequest().withApplicationId( appId ).withCollection( collectionName );
+
+ return executeResumeAndCreateResponse( payload, request, callback );
+ }
+
+
+ @RequireSystemAccess
+ @POST
+ @Path( "rebuild/management" )
+ public JSONWithPadding rebuildInternalIndexesPost(
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback ) throws Exception {
+
+
+ final UUID managementAppId = emf.getManagementAppId();
+
+ logger.info( "Rebuilding management application with id {} ", managementAppId );
+ final ReIndexRequestBuilder request = createRequest().withApplicationId( managementAppId );
+
+ return executeAndCreateResponse( request, callback );
+ }
+
+
+ @RequireSystemAccess
+ @PUT
+ @Path( "rebuild/management" )
+ public JSONWithPadding rebuildInternalIndexesPut( final Map<String, Object> payload,
+ @QueryParam( "callback" ) @DefaultValue( "callback" )
+ String callback ) throws Exception {
+
+
+ final UUID managementAppId = emf.getManagementAppId();
+
+ logger.info( "Resuming rebuilding management application with id {} ", managementAppId );
+ final ReIndexRequestBuilder request = createRequest().withApplicationId( managementAppId );
+
+ return executeResumeAndCreateResponse( payload, request, callback );
+ }
+
+
+ @RequireSystemAccess
+ @POST
+ @Path(RootResource.APPLICATION_ID_PATH)
+ public JSONWithPadding addIndex( @Context UriInfo ui,
+ @PathParam( "applicationId" ) final String applicationIdStr,
+ Map<String, Object> config,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+ throws Exception {
+
+ Preconditions
+ .checkNotNull( config, "Payload for config is null, please pass {replicas:int, shards:int} in body" );
+
+ ApiResponse response = createApiResponse();
+
+ if ( !config.containsKey( "replicas" ) || !config.containsKey( "shards" ) ||
+ !( config.get( "replicas" ) instanceof Integer ) || !( config.get( "shards" ) instanceof Integer ) ) {
+ throw new IllegalArgumentException( "body must contains 'replicas' of type int and 'shards' of type int" );
+ }
+
+ if ( !config.containsKey( "indexSuffix" ) ) {
+ throw new IllegalArgumentException( "Please add an indexSuffix to your post" );
+ }
+ final UUID appId = UUIDUtils.tryExtractUUID( applicationIdStr );
+
+ if(appId == null){
+ throw new IllegalArgumentException("app id was not parsed");
+ }
+
+ EntityManager em = emf.getEntityManager(appId);
+ em.addIndex(config.get("indexSuffix").toString(), (int) config.get("shards"),
+ (int) config.get("replicas"), (String) config.get("writeConsistency"));
+ response.setAction( "Add index to alias" );
+
+ return new JSONWithPadding( response, callback );
+ }
+
+
+ private ReIndexService getReIndexService() {
+ return injector.getInstance( ReIndexService.class );
+ }
+
+
+ private ReIndexRequestBuilder createRequest() {
+ //TODO: wire this up through spring, and in the future guice.
+ return new ReIndexRequestBuilderImpl();
+ }
+
+
+ private JSONWithPadding executeResumeAndCreateResponse( final Map<String, Object> payload,
+ final ReIndexRequestBuilder request,
+ final String callback ) {
+
+ Map<String,Object> newPayload = payload;
+ if(newPayload == null || !payload.containsKey( UPDATED_FIELD )){
+ newPayload = new HashMap<>(1);
+ newPayload.put(UPDATED_FIELD,0);
+ }
+
+ Preconditions.checkArgument(newPayload.get(UPDATED_FIELD) instanceof Number,
+ "You must specified the field \"updated\" in the payload and it must be a timestamp" );
+
+ //add our updated timestamp to the request
+ if ( newPayload.containsKey( UPDATED_FIELD ) ) {
+ final long timestamp = ConversionUtils.getLong(newPayload.get(UPDATED_FIELD));
+ request.withStartTimestamp( timestamp );
+ }
+
+ return executeAndCreateResponse( request, callback );
+ }
+
+
+ /**
+ * Execute the request and return the response.
+ */
+ private JSONWithPadding executeAndCreateResponse( final ReIndexRequestBuilder request, final String callback ) {
+
+
+ final ReIndexService.ReIndexStatus status = getReIndexService().rebuildIndex( request );
+
+ final ApiResponse response = createApiResponse();
+
+ response.setAction( "rebuild indexes" );
+ response.setProperty( "jobId", status.getJobId() );
+ response.setProperty( "status", status.getStatus() );
+ response.setProperty( "lastUpdatedEpoch", status.getLastUpdated() );
+ response.setProperty( "numberQueued", status.getNumberProcessed() );
+ response.setSuccess();
+
+ return new JSONWithPadding( response, callback );
+ }
+}
[11/50] [abbrv] usergrid git commit: Change read repair to interact
with c* directly and only fire and index operation message to get the ES
document removed from all regions.
Posted by sn...@apache.org.
Change read repair to interact with c* directly and only fire and index operation message to get the ES document removed from all regions.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/1b43bda3
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/1b43bda3
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/1b43bda3
Branch: refs/heads/asf-site
Commit: 1b43bda3f801172b0e59f927ff3ae52a559d36cc
Parents: 70d7a95
Author: Michael Russo <mi...@gmail.com>
Authored: Tue Oct 27 10:50:16 2015 -0700
Committer: Michael Russo <mi...@gmail.com>
Committed: Tue Oct 27 12:56:13 2015 -0700
----------------------------------------------------------------------
.../asyncevents/AsyncEventService.java | 5 ++
.../asyncevents/InMemoryAsyncEventService.java | 5 ++
.../read/traverse/AbstractReadGraphFilter.java | 69 +++++++++++++++++---
.../traverse/ReadGraphCollectionFilter.java | 10 ++-
.../traverse/ReadGraphConnectionFilter.java | 10 ++-
.../impl/stage/NodeDeleteListenerImpl.java | 27 ++++----
6 files changed, 97 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/1b43bda3/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventService.java
index dcfffcb..dbf8996 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventService.java
@@ -77,6 +77,11 @@ public interface AsyncEventService extends ReIndexAction {
*/
void queueEntityDelete(final ApplicationScope applicationScope, final Id entityId);
+ /**
+ *
+ * @param indexOperationMessage
+ */
+ void queueIndexOperationMessage( final IndexOperationMessage indexOperationMessage );
/**
* current queue depth
http://git-wip-us.apache.org/repos/asf/usergrid/blob/1b43bda3/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/InMemoryAsyncEventService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/InMemoryAsyncEventService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/InMemoryAsyncEventService.java
index fc6385c..d8334b3 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/InMemoryAsyncEventService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/InMemoryAsyncEventService.java
@@ -105,6 +105,11 @@ public class InMemoryAsyncEventService implements AsyncEventService {
run( results.getCompactedNode() );
}
+ @Override
+ public void queueIndexOperationMessage(final IndexOperationMessage indexOperationMessage){
+ //this is not used locally
+ }
+
public void index( final ApplicationScope applicationScope, final Id id, final long updatedSince ) {
final EntityIndexOperation entityIndexOperation = new EntityIndexOperation( applicationScope, id, updatedSince );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/1b43bda3/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java
index 9d050c8..89230d7 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java
@@ -20,7 +20,11 @@
package org.apache.usergrid.corepersistence.pipeline.read.traverse;
+import org.apache.usergrid.corepersistence.asyncevents.AsyncEventService;
+import org.apache.usergrid.corepersistence.asyncevents.EventBuilder;
+import org.apache.usergrid.corepersistence.asyncevents.EventBuilderImpl;
import org.apache.usergrid.persistence.core.rx.RxTaskScheduler;
+import org.apache.usergrid.persistence.index.impl.IndexOperationMessage;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -42,6 +46,7 @@ import com.google.common.base.Optional;
import rx.Observable;
+
/**
* Command for reading graph edges
*/
@@ -51,15 +56,21 @@ public abstract class AbstractReadGraphFilter extends AbstractPathFilter<Id, Id,
private final GraphManagerFactory graphManagerFactory;
private final RxTaskScheduler rxTaskScheduler;
+ private final EventBuilder eventBuilder;
+ private final AsyncEventService asyncEventService;
/**
* Create a new instance of our command
*/
public AbstractReadGraphFilter( final GraphManagerFactory graphManagerFactory,
- final RxTaskScheduler rxTaskScheduler) {
+ final RxTaskScheduler rxTaskScheduler,
+ final EventBuilder eventBuilder,
+ final AsyncEventService asyncEventService ) {
this.graphManagerFactory = graphManagerFactory;
this.rxTaskScheduler = rxTaskScheduler;
+ this.eventBuilder = eventBuilder;
+ this.asyncEventService = asyncEventService;
}
@@ -107,28 +118,56 @@ public abstract class AbstractReadGraphFilter extends AbstractPathFilter<Id, Id,
final boolean isTargetNodeDelete = markedEdge.isTargetNodeDeleted();
+ if (isDeleted) {
- if(isDeleted){
logger.trace("Edge {} is deleted, deleting the edge", markedEdge);
- graphManager.deleteEdge(markedEdge).subscribeOn(rxTaskScheduler.getAsyncIOScheduler())
+ final Observable<IndexOperationMessage> indexMessageObservable = eventBuilder.buildDeleteEdge(applicationScope, markedEdge);
+
+ indexMessageObservable
+ .compose(applyCollector())
+ .subscribeOn(rxTaskScheduler.getAsyncIOScheduler())
.subscribe();
+
}
- if(isSourceNodeDeleted){
- final Id sourceNodeId = markedEdge.getSourceNode();
+ if (isSourceNodeDeleted) {
+ final Id sourceNodeId = markedEdge.getSourceNode();
logger.trace("Edge {} has a deleted source node, deleting the entity for id {}", markedEdge, sourceNodeId);
- graphManager.compactNode(sourceNodeId).subscribeOn(rxTaskScheduler.getAsyncIOScheduler())
+
+ final EventBuilderImpl.EntityDeleteResults
+ entityDeleteResults = eventBuilder.buildEntityDelete(applicationScope, sourceNodeId);
+
+ entityDeleteResults.getIndexObservable()
+ .compose(applyCollector())
+ .subscribeOn(rxTaskScheduler.getAsyncIOScheduler())
.subscribe();
+
+ Observable.merge(entityDeleteResults.getEntitiesDeleted(),
+ entityDeleteResults.getCompactedNode())
+ .subscribeOn(rxTaskScheduler.getAsyncIOScheduler()).
+ subscribe();
+
}
- if(isTargetNodeDelete){
+ if (isTargetNodeDelete) {
final Id targetNodeId = markedEdge.getTargetNode();
+ logger.trace("Edge {} has a deleted target node, deleting the entity for id {}", markedEdge, targetNodeId);
- logger.trace("Edge {} has a deleted target node, deleting the entity for id {}", markedEdge, targetNodeId );
- graphManager.compactNode(targetNodeId).subscribeOn(rxTaskScheduler.getAsyncIOScheduler())
+ final EventBuilderImpl.EntityDeleteResults
+ entityDeleteResults = eventBuilder.buildEntityDelete(applicationScope, targetNodeId);
+
+ entityDeleteResults.getIndexObservable()
+ .compose(applyCollector())
+ .subscribeOn(rxTaskScheduler.getAsyncIOScheduler())
.subscribe();
+
+ Observable.merge(entityDeleteResults.getEntitiesDeleted(),
+ entityDeleteResults.getCompactedNode())
+ .subscribeOn(rxTaskScheduler.getAsyncIOScheduler()).
+ subscribe();
+
}
@@ -202,4 +241,16 @@ public abstract class AbstractReadGraphFilter extends AbstractPathFilter<Id, Id,
return cursorEdge;
}
}
+
+ private Observable.Transformer<IndexOperationMessage, IndexOperationMessage> applyCollector() {
+
+ return observable -> observable
+ .filter((IndexOperationMessage msg) -> !msg.isEmpty())
+ .collect(() -> new IndexOperationMessage(), (collector, single) -> collector.ingest(single))
+ .doOnNext(indexOperation -> {
+ asyncEventService.queueIndexOperationMessage(indexOperation);
+ });
+
+ }
+
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/1b43bda3/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/ReadGraphCollectionFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/ReadGraphCollectionFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/ReadGraphCollectionFilter.java
index 1d63bc6..3d7df3b 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/ReadGraphCollectionFilter.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/ReadGraphCollectionFilter.java
@@ -20,6 +20,8 @@
package org.apache.usergrid.corepersistence.pipeline.read.traverse;
+import org.apache.usergrid.corepersistence.asyncevents.AsyncEventService;
+import org.apache.usergrid.corepersistence.asyncevents.EventBuilder;
import org.apache.usergrid.persistence.core.rx.RxTaskScheduler;
import org.apache.usergrid.persistence.graph.GraphManagerFactory;
@@ -41,8 +43,12 @@ public class ReadGraphCollectionFilter extends AbstractReadGraphFilter {
* Create a new instance of our command
*/
@Inject
- public ReadGraphCollectionFilter( final GraphManagerFactory graphManagerFactory, final RxTaskScheduler rxTaskScheduler, @Assisted final String collectionName ) {
- super( graphManagerFactory, rxTaskScheduler );
+ public ReadGraphCollectionFilter( final GraphManagerFactory graphManagerFactory,
+ final RxTaskScheduler rxTaskScheduler,
+ final EventBuilder eventBuilder,
+ final AsyncEventService asyncEventService,
+ @Assisted final String collectionName ) {
+ super( graphManagerFactory, rxTaskScheduler, eventBuilder, asyncEventService );
this.collectionName = collectionName;
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/1b43bda3/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/ReadGraphConnectionFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/ReadGraphConnectionFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/ReadGraphConnectionFilter.java
index efe94db..b2d368b 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/ReadGraphConnectionFilter.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/ReadGraphConnectionFilter.java
@@ -20,6 +20,8 @@
package org.apache.usergrid.corepersistence.pipeline.read.traverse;
+import org.apache.usergrid.corepersistence.asyncevents.AsyncEventService;
+import org.apache.usergrid.corepersistence.asyncevents.EventBuilder;
import org.apache.usergrid.persistence.core.rx.RxTaskScheduler;
import org.apache.usergrid.persistence.graph.GraphManagerFactory;
@@ -41,8 +43,12 @@ public class ReadGraphConnectionFilter extends AbstractReadGraphFilter {
* Create a new instance of our command
*/
@Inject
- public ReadGraphConnectionFilter( final GraphManagerFactory graphManagerFactory, final RxTaskScheduler rxTaskScheduler, @Assisted final String connectionName ) {
- super( graphManagerFactory, rxTaskScheduler );
+ public ReadGraphConnectionFilter( final GraphManagerFactory graphManagerFactory,
+ final RxTaskScheduler rxTaskScheduler,
+ final EventBuilder eventBuilder,
+ final AsyncEventService asyncEventService,
+ @Assisted final String connectionName ) {
+ super( graphManagerFactory, rxTaskScheduler, eventBuilder, asyncEventService );
this.connectionName = connectionName;
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/1b43bda3/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/NodeDeleteListenerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/NodeDeleteListenerImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/NodeDeleteListenerImpl.java
index e4eb5fc..343cc77 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/NodeDeleteListenerImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/NodeDeleteListenerImpl.java
@@ -157,26 +157,26 @@ public class NodeDeleteListenerImpl implements NodeDeleteListener {
//get all edges pointing to the target node and buffer then into groups for deletion
Observable<MarkedEdge> targetEdges =
- getEdgesTypesToTarget( scope, new SimpleSearchEdgeType( node, null, null ) )
- .subscribeOn( Schedulers.io() ).flatMap( edgeType -> Observable.create( new ObservableIterator<MarkedEdge>( "getTargetEdges" ) {
+ getEdgesTypesToTarget(scope, new SimpleSearchEdgeType(node, null, null))
+ .flatMap(edgeType -> Observable.create(new ObservableIterator<MarkedEdge>("getTargetEdges") {
@Override
protected Iterator<MarkedEdge> getIterator() {
- return storageSerialization.getEdgesToTarget( scope,
- new SimpleSearchByEdgeType( node, edgeType, maxVersion, SearchByEdgeType.Order.DESCENDING, Optional.<Edge>absent() ) );
+ return storageSerialization.getEdgesToTarget(scope,
+ new SimpleSearchByEdgeType(node, edgeType, maxVersion, SearchByEdgeType.Order.DESCENDING, Optional.<Edge>absent()));
}
- } ) );
+ }));
//get all edges pointing to the source node and buffer them into groups for deletion
Observable<MarkedEdge> sourceEdges =
- getEdgesTypesFromSource( scope, new SimpleSearchEdgeType( node, null, null ) )
- .subscribeOn( Schedulers.io() ).flatMap( edgeType -> Observable.create( new ObservableIterator<MarkedEdge>( "getSourceEdges" ) {
+ getEdgesTypesFromSource(scope, new SimpleSearchEdgeType(node, null, null))
+ .flatMap(edgeType -> Observable.create(new ObservableIterator<MarkedEdge>("getSourceEdges") {
@Override
protected Iterator<MarkedEdge> getIterator() {
- return storageSerialization.getEdgesFromSource( scope,
- new SimpleSearchByEdgeType( node, edgeType, maxVersion, SearchByEdgeType.Order.DESCENDING, Optional.<Edge>absent() ) );
+ return storageSerialization.getEdgesFromSource(scope,
+ new SimpleSearchByEdgeType(node, edgeType, maxVersion, SearchByEdgeType.Order.DESCENDING, Optional.<Edge>absent()));
}
- } ) );
+ }));
//merge both source and target into 1 observable. We'll need to check them all regardless of order
return Observable.merge( targetEdges, sourceEdges )
@@ -235,12 +235,7 @@ public class NodeDeleteListenerImpl implements NodeDeleteListener {
//run both the source/target edge type cleanup, then proceed
return Observable.merge( sourceMetaCleanup, targetMetaCleanup ).lastOrDefault( null )
- .flatMap( new Func1<Integer, Observable<MarkedEdge>>() {
- @Override
- public Observable<MarkedEdge> call( final Integer integer ) {
- return Observable.from( markedEdges );
- }
- } );
+ .flatMap(integer -> Observable.from( markedEdges ));
} );
}
[19/50] [abbrv] usergrid git commit: USERGRID-1044: don't log UUID
mismatch if UUID was never retrieved during load
Posted by sn...@apache.org.
USERGRID-1044: don't log UUID mismatch if UUID was never retrieved during load
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/0fc18041
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/0fc18041
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/0fc18041
Branch: refs/heads/asf-site
Commit: 0fc18041e32df049040f26a0faaef1f46dbe6d9f
Parents: 2a09bf1
Author: Mike Dunker <md...@apigee.com>
Authored: Wed Oct 28 08:09:19 2015 -0700
Committer: Mike Dunker <md...@apigee.com>
Committed: Wed Oct 28 08:09:19 2015 -0700
----------------------------------------------------------------------
.../scala/org/apache/usergrid/scenarios/AuditScenarios.scala | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/0fc18041/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala
index d1e2c4b..cc59e0a 100644
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala
@@ -183,7 +183,7 @@ object AuditScenarios {
errorString += "|NoUuidReturned"
Settings.incAuditPayloadUuidError()
println(s"PAYLOAD UUID MISSING (DIRECT): requestedUuid=$uuid")
- } else if (!uuid.equalsIgnoreCase(entityUuid)) {
+ } else if (!uuid.isEmpty && !uuid.equalsIgnoreCase(entityUuid)) {
errorString += "|ReturnedUuidMismatch"
Settings.incAuditPayloadUuidError()
println(s"PAYLOAD UUID MISMATCH (DIRECT): requestedUuid=$uuid returnedUuid=$entityUuid")
@@ -247,7 +247,7 @@ object AuditScenarios {
errorString += "|NoUuidReturned"
Settings.incAuditPayloadUuidError()
println(s"PAYLOAD UUID MISSING (QUERY): requestedUuid=$uuid")
- } else if (!uuid.equalsIgnoreCase(entityUuid)) {
+ } else if (!uuid.isEmpty && !uuid.equalsIgnoreCase(entityUuid)) {
errorString += "|ReturnedUuidMismatch"
Settings.incAuditPayloadUuidError()
println(s"PAYLOAD UUID MISMATCH (QUERY): requestedUuid=$uuid returnedUuid=$entityUuid")
[21/50] [abbrv] usergrid git commit: Fix failing test to ensure we
don't set the same timestamp for all entities inserted into the index (since
we don't do actually do that). Also ensure it's a long as what's required of
the timestamps
Posted by sn...@apache.org.
Fix failing test to ensure we don't set the same timestamp for all entities inserted into the index (since we don't do actually do that). Also ensure it's a long as what's required of the timestamps
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/c652171f
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/c652171f
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/c652171f
Branch: refs/heads/asf-site
Commit: c652171f476f6ad953c94328e0fb2f4ea0b3aa44
Parents: 22beca2
Author: Michael Russo <mi...@gmail.com>
Authored: Wed Oct 28 16:56:19 2015 -0700
Committer: Michael Russo <mi...@gmail.com>
Committed: Wed Oct 28 16:56:19 2015 -0700
----------------------------------------------------------------------
.../persistence/index/impl/EntityIndexTest.java | 14 ++++++--------
1 file changed, 6 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c652171f/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
index 5243d5a..d6758dd 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
@@ -31,6 +31,7 @@ import java.util.concurrent.atomic.AtomicLong;
import com.google.common.base.Optional;
import org.apache.usergrid.persistence.core.astyanax.CassandraFig;
import org.apache.usergrid.persistence.index.*;
+import org.apache.usergrid.persistence.model.field.*;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
@@ -48,10 +49,6 @@ import org.apache.usergrid.persistence.index.utils.UUIDUtils;
import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.entity.SimpleId;
-import org.apache.usergrid.persistence.model.field.ArrayField;
-import org.apache.usergrid.persistence.model.field.IntegerField;
-import org.apache.usergrid.persistence.model.field.StringField;
-import org.apache.usergrid.persistence.model.field.UUIDField;
import org.apache.usergrid.persistence.model.util.EntityUtils;
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
@@ -394,7 +391,6 @@ public class EntityIndexTest extends BaseIT {
int numberOfEntities = 1000;
int versionToSearchFor = numberOfEntities / 2;
- IndexEdge searchEdge = new IndexEdgeImpl( appId, "mehCars", SearchEdge.NodeType.SOURCE, 1 );
UUID entityUUID = UUID.randomUUID();
Id entityId = new SimpleId( "mehCar" );
@@ -408,9 +404,11 @@ public class EntityIndexTest extends BaseIT {
Entity[] entity = new Entity[numberOfEntities];
for(int i = 0; i < numberOfEntities; i++) {
entity[i] = EntityIndexMapUtils.fromMap( entityMap );
- EntityUtils.setId( entity[i], entityId );
- EntityUtils.setVersion( entity[i], UUIDGenerator.newTimeUUID() );
- entity[i].setField( new UUIDField( IndexingUtils.ENTITY_ID_FIELDNAME, entityUUID ) );
+ EntityUtils.setId(entity[i], entityId);
+ EntityUtils.setVersion(entity[i], UUIDGenerator.newTimeUUID());
+ entity[i].setField(new UUIDField(IndexingUtils.ENTITY_ID_FIELDNAME, entityUUID));
+
+ IndexEdge searchEdge = new IndexEdgeImpl( appId, "mehCars", SearchEdge.NodeType.SOURCE, System.currentTimeMillis()*1000 );
//index the new entity. This is where the loop will be set to create like 100 entities.
indexProducer.put(entityIndex.createBatch().index( searchEdge, entity[i] ).build()).subscribe();
[40/50] [abbrv] usergrid git commit: Fixes race condition in test
Posted by sn...@apache.org.
Fixes race condition in test
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/b031d4d3
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/b031d4d3
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/b031d4d3
Branch: refs/heads/asf-site
Commit: b031d4d3da09ee65dcde5ab7521627bb2782c9ea
Parents: b422364
Author: Todd Nine <tn...@apigee.com>
Authored: Fri Oct 30 17:32:19 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Fri Oct 30 17:32:19 2015 -0600
----------------------------------------------------------------------
.../java/org/apache/usergrid/corepersistence/index/RxTest.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/b031d4d3/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/RxTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/RxTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/RxTest.java
index d7b0bdb..ce78b15 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/RxTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/RxTest.java
@@ -45,10 +45,10 @@ public class RxTest {
final int count = 10;
- final CountDownLatch latch = new CountDownLatch( count );
+ final CountDownLatch latch = new CountDownLatch( count+1 );
final Subscription connectedObservable =
- Observable.range( 0, count ).doOnNext( integer -> latch.countDown() ).subscribeOn( Schedulers.io() )
+ Observable.range( 0, count ).doOnNext( integer -> latch.countDown() ).doOnCompleted( () -> latch.countDown() ).subscribeOn( Schedulers.io() )
.subscribe();
[09/50] [abbrv] usergrid git commit: USERGRID-1044: handle audit for
CSV rows that don't have UUIDs by accessing via name
Posted by sn...@apache.org.
USERGRID-1044: handle audit for CSV rows that don't have UUIDs by accessing via name
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/9158772b
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/9158772b
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/9158772b
Branch: refs/heads/asf-site
Commit: 9158772bc20befa8affa6c1bc10af032a07d0400
Parents: 4b2d003
Author: Mike Dunker <md...@apigee.com>
Authored: Tue Oct 27 12:46:48 2015 -0700
Committer: Mike Dunker <md...@apigee.com>
Committed: Tue Oct 27 12:46:48 2015 -0700
----------------------------------------------------------------------
.../org/apache/usergrid/datagenerators/FeederGenerator.scala | 5 ++++-
.../scala/org/apache/usergrid/scenarios/AuditScenarios.scala | 4 ++--
2 files changed, 6 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/9158772b/stack/loadtests/src/main/scala/org/apache/usergrid/datagenerators/FeederGenerator.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/datagenerators/FeederGenerator.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/datagenerators/FeederGenerator.scala
index b8a28d5..73b3d3c 100755
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/datagenerators/FeederGenerator.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/datagenerators/FeederGenerator.scala
@@ -280,8 +280,11 @@ object FeederGenerator {
val uuid = if (line != null) array(2) else ""
val modified = if (line != null) array(3) else ""
//println(s"$collectionName|$name|$uuid|$modified")
+ val accessField = if (uuid != "") uuid else name
+ val queryField = if (uuid != "") s"uuid='$uuid'" else s"name='$name'"
- Map("collectionName" -> collectionName, "name" -> name, "uuid" -> uuid, "modified" -> modified, "validEntity" -> validEntity)
+ Map("collectionName" -> collectionName, "name" -> name, "uuid" -> uuid, "modified" -> modified,
+ "validEntity" -> validEntity, "accessField" -> accessField, "queryField" -> queryField)
}
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/9158772b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala
index f90348e..7a93eb6 100644
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala
@@ -144,7 +144,7 @@ object AuditScenarios {
val getCollectionEntityDirect = exec(
http("GET collection entity direct")
- .get("/${collectionName}/${uuid}")
+ .get("/${collectionName}/${accessField}")
.headers(Headers.authToken)
.headers(Headers.usergridRegionHeaders)
.check()
@@ -213,7 +213,7 @@ object AuditScenarios {
val getCollectionEntity = exec(
http("GET collection entity")
- .get("/${collectionName}?ql=uuid=${uuid}")
+ .get("/${collectionName}?ql=${queryField}")
.headers(Headers.authToken)
.headers(Headers.usergridRegionHeaders)
.check(status.is(200), status.saveAs(SessionVarStatus), jsonPath("$.count").optional.saveAs("count"),
[29/50] [abbrv] usergrid git commit: Ensure that status is updated
properly.
Posted by sn...@apache.org.
Ensure that status is updated properly.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/f8c703c0
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/f8c703c0
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/f8c703c0
Branch: refs/heads/asf-site
Commit: f8c703c02c1182ad63ad86587749eb1ae09c202a
Parents: 471dc35
Author: Dave Johnson <sn...@apache.org>
Authored: Thu Oct 29 17:57:32 2015 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Thu Oct 29 17:57:32 2015 -0400
----------------------------------------------------------------------
.../rest/system/ConnectionResource.java | 48 +++++++++++++-------
1 file changed, 32 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f8c703c0/stack/rest/src/main/java/org/apache/usergrid/rest/system/ConnectionResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/system/ConnectionResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/system/ConnectionResource.java
index 6e683ed..14b79f3 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/system/ConnectionResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/system/ConnectionResource.java
@@ -55,6 +55,7 @@ import com.google.common.base.Preconditions;
import com.sun.jersey.api.json.JSONWithPadding;
import rx.Observable;
+import rx.functions.Action1;
import rx.schedulers.Schedulers;
@@ -146,36 +147,51 @@ public class ConnectionResource extends AbstractContextResource {
//start de duping and run in the background
connectionService.deDupeConnections( applicationScopeObservable ).buffer( 10, TimeUnit.SECONDS, 1000 )
- .doOnNext( buffer -> {
+ .doOnNext(buffer -> {
- final long runningTotal = count.addAndGet( buffer.size() );
+ final long runningTotal = count.addAndGet(buffer.size());
final Map<String, Object> status = new HashMap<String, Object>() {{
- put( "countProcessed", runningTotal );
- put( "updatedTimestamp", System.currentTimeMillis() );
+ put("countProcessed", runningTotal);
+ put("updatedTimestamp", System.currentTimeMillis());
}};
- statusService.setStatus( CpNamingUtils.MANAGEMENT_APPLICATION_ID, jobId,
- StatusService.Status.INPROGRESS, status ).toBlocking().lastOrDefault( null );
- } ).doOnSubscribe( () -> {
- statusService.setStatus( CpNamingUtils.MANAGEMENT_APPLICATION_ID, jobId, StatusService.Status.STARTED,
- new HashMap<>() ).toBlocking().lastOrDefault( null );
- } ).doOnCompleted( () -> {
+ statusService.setStatus(CpNamingUtils.MANAGEMENT_APPLICATION_ID, jobId,
+ StatusService.Status.INPROGRESS, status).toBlocking().lastOrDefault(null);
+ }).doOnSubscribe(() -> {
+
+ statusService.setStatus(CpNamingUtils.MANAGEMENT_APPLICATION_ID,
+ jobId, StatusService.Status.STARTED, new HashMap<>()).toBlocking().lastOrDefault(null);
+
+ }).doOnCompleted(() -> {
final long runningTotal = count.get();
final Map<String, Object> status = new HashMap<String, Object>() {{
- put( "countProcessed", runningTotal );
- put( "updatedTimestamp", System.currentTimeMillis() );
+ put("countProcessed", runningTotal);
+ put("updatedTimestamp", System.currentTimeMillis());
}};
- statusService
- .setStatus( CpNamingUtils.MANAGEMENT_APPLICATION_ID, jobId, StatusService.Status.COMPLETE, status );
- } ).subscribeOn( Schedulers.newThread() ).subscribe();
+ statusService.setStatus(CpNamingUtils.MANAGEMENT_APPLICATION_ID,
+ jobId, StatusService.Status.COMPLETE, status).toBlocking().lastOrDefault(null);
+
+ }).doOnError( (throwable) -> {
+ logger.error("Error deduping connections", throwable);
+
+ final Map<String, Object> status = new HashMap<String, Object>() {{
+ put("error", throwable.getMessage() );
+ }};
+
+ statusService.setStatus(CpNamingUtils.MANAGEMENT_APPLICATION_ID,
+ jobId, StatusService.Status.FAILED, status).toBlocking().lastOrDefault(null);;
+
+ } ).subscribeOn(Schedulers.newThread()).subscribe();
+
+ final StatusService.JobStatus status =
+ new StatusService.JobStatus( jobId, StatusService.Status.STARTED, new HashMap<>( ) );
- final StatusService.JobStatus status = new StatusService.JobStatus( jobId, StatusService.Status.STARTED, new HashMap<>( ) );
return createResult( status, callback );
}
[46/50] [abbrv] usergrid git commit: This closes #163 - cannot merge
from unknown repository
Posted by sn...@apache.org.
This closes #163 - cannot merge from unknown repository
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/61a25bdd
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/61a25bdd
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/61a25bdd
Branch: refs/heads/asf-site
Commit: 61a25bddfe5c24ffb750a089e436732ee0d19604
Parents: 11594ce
Author: Mike Dunker <md...@apigee.com>
Authored: Tue Nov 3 14:52:53 2015 -0800
Committer: Mike Dunker <md...@apigee.com>
Committed: Tue Nov 3 14:52:53 2015 -0800
----------------------------------------------------------------------
----------------------------------------------------------------------
[02/50] [abbrv] usergrid git commit: Make the graph read repair
directly compact nodes in graph instead of queueing events. Misc prop file
changes.
Posted by sn...@apache.org.
Make the graph read repair directly compact nodes in graph instead of queueing events. Misc prop file changes.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/70d7a958
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/70d7a958
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/70d7a958
Branch: refs/heads/asf-site
Commit: 70d7a9586ece0f32ec5aa50334cd4d70f440b2c6
Parents: 5eed978
Author: Michael Russo <mi...@gmail.com>
Authored: Mon Oct 26 22:26:07 2015 -0700
Committer: Michael Russo <mi...@gmail.com>
Committed: Mon Oct 26 22:26:07 2015 -0700
----------------------------------------------------------------------
.../index/IndexProcessorFig.java | 4 ++--
.../read/traverse/AbstractReadGraphFilter.java | 25 ++++++++++----------
.../traverse/ReadGraphCollectionFilter.java | 6 ++---
.../traverse/ReadGraphConnectionFilter.java | 6 ++---
4 files changed, 21 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/70d7a958/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java
index ec9b315..7650c62 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java
@@ -65,14 +65,14 @@ public interface IndexProcessorFig extends GuicyFig {
* Received messages will remain 'in flight' until they are ack'd(deleted) or this timeout occurs.
* If the timeout occurs, the messages will become visible again for re-processing.
*/
- @Default( "5000" ) // 5 seconds
+ @Default( "30000" ) // 30 seconds
@Key( INDEX_QUEUE_VISIBILITY_TIMEOUT )
int getIndexQueueVisibilityTimeout();
/**
* The number of worker threads used to read index write requests from the queue.
*/
- @Default( "8" )
+ @Default( "16" )
@Key( ELASTICSEARCH_WORKER_COUNT )
int getWorkerCount();
http://git-wip-us.apache.org/repos/asf/usergrid/blob/70d7a958/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java
index 621edd2..9d050c8 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java
@@ -20,10 +20,10 @@
package org.apache.usergrid.corepersistence.pipeline.read.traverse;
+import org.apache.usergrid.persistence.core.rx.RxTaskScheduler;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.usergrid.corepersistence.asyncevents.AsyncEventService;
import org.apache.usergrid.corepersistence.pipeline.cursor.CursorSerializer;
import org.apache.usergrid.corepersistence.pipeline.read.AbstractPathFilter;
import org.apache.usergrid.corepersistence.pipeline.read.EdgePath;
@@ -50,16 +50,16 @@ public abstract class AbstractReadGraphFilter extends AbstractPathFilter<Id, Id,
private static final Logger logger = LoggerFactory.getLogger( AbstractReadGraphFilter.class );
private final GraphManagerFactory graphManagerFactory;
- private final AsyncEventService asyncEventService;
+ private final RxTaskScheduler rxTaskScheduler;
/**
* Create a new instance of our command
*/
public AbstractReadGraphFilter( final GraphManagerFactory graphManagerFactory,
- final AsyncEventService asyncEventService ) {
+ final RxTaskScheduler rxTaskScheduler) {
this.graphManagerFactory = graphManagerFactory;
- this.asyncEventService = asyncEventService;
+ this.rxTaskScheduler = rxTaskScheduler;
}
@@ -109,25 +109,26 @@ public abstract class AbstractReadGraphFilter extends AbstractPathFilter<Id, Id,
if(isDeleted){
- logger.trace( "Edge {} is deleted, queueing the delete event", markedEdge );
- asyncEventService.queueDeleteEdge( applicationScope, markedEdge );
+ logger.trace("Edge {} is deleted, deleting the edge", markedEdge);
+ graphManager.deleteEdge(markedEdge).subscribeOn(rxTaskScheduler.getAsyncIOScheduler())
+ .subscribe();
}
if(isSourceNodeDeleted){
final Id sourceNodeId = markedEdge.getSourceNode();
- logger.trace( "Edge {} has a deleted source node, queueing the delete entity event for id {}", markedEdge, sourceNodeId );
-
- asyncEventService.queueEntityDelete( applicationScope, sourceNodeId );
+ logger.trace("Edge {} has a deleted source node, deleting the entity for id {}", markedEdge, sourceNodeId);
+ graphManager.compactNode(sourceNodeId).subscribeOn(rxTaskScheduler.getAsyncIOScheduler())
+ .subscribe();
}
if(isTargetNodeDelete){
final Id targetNodeId = markedEdge.getTargetNode();
- logger.trace( "Edge {} has a deleted target node, queueing the delete entity event for id {}", markedEdge, targetNodeId );
-
- asyncEventService.queueEntityDelete( applicationScope, targetNodeId );
+ logger.trace("Edge {} has a deleted target node, deleting the entity for id {}", markedEdge, targetNodeId );
+ graphManager.compactNode(targetNodeId).subscribeOn(rxTaskScheduler.getAsyncIOScheduler())
+ .subscribe();
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/70d7a958/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/ReadGraphCollectionFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/ReadGraphCollectionFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/ReadGraphCollectionFilter.java
index db5a0a8..1d63bc6 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/ReadGraphCollectionFilter.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/ReadGraphCollectionFilter.java
@@ -20,7 +20,7 @@
package org.apache.usergrid.corepersistence.pipeline.read.traverse;
-import org.apache.usergrid.corepersistence.asyncevents.AsyncEventService;
+import org.apache.usergrid.persistence.core.rx.RxTaskScheduler;
import org.apache.usergrid.persistence.graph.GraphManagerFactory;
import com.google.inject.Inject;
@@ -41,8 +41,8 @@ public class ReadGraphCollectionFilter extends AbstractReadGraphFilter {
* Create a new instance of our command
*/
@Inject
- public ReadGraphCollectionFilter( final GraphManagerFactory graphManagerFactory, final AsyncEventService asyncEventService, @Assisted final String collectionName ) {
- super( graphManagerFactory, asyncEventService );
+ public ReadGraphCollectionFilter( final GraphManagerFactory graphManagerFactory, final RxTaskScheduler rxTaskScheduler, @Assisted final String collectionName ) {
+ super( graphManagerFactory, rxTaskScheduler );
this.collectionName = collectionName;
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/70d7a958/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/ReadGraphConnectionFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/ReadGraphConnectionFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/ReadGraphConnectionFilter.java
index 93e8fd4..efe94db 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/ReadGraphConnectionFilter.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/ReadGraphConnectionFilter.java
@@ -20,7 +20,7 @@
package org.apache.usergrid.corepersistence.pipeline.read.traverse;
-import org.apache.usergrid.corepersistence.asyncevents.AsyncEventService;
+import org.apache.usergrid.persistence.core.rx.RxTaskScheduler;
import org.apache.usergrid.persistence.graph.GraphManagerFactory;
import com.google.inject.Inject;
@@ -41,8 +41,8 @@ public class ReadGraphConnectionFilter extends AbstractReadGraphFilter {
* Create a new instance of our command
*/
@Inject
- public ReadGraphConnectionFilter( final GraphManagerFactory graphManagerFactory, final AsyncEventService asyncEventService, @Assisted final String connectionName ) {
- super( graphManagerFactory, asyncEventService );
+ public ReadGraphConnectionFilter( final GraphManagerFactory graphManagerFactory, final RxTaskScheduler rxTaskScheduler, @Assisted final String connectionName ) {
+ super( graphManagerFactory, rxTaskScheduler );
this.connectionName = connectionName;
}
[14/50] [abbrv] usergrid git commit: USERGRID-1044: fixes
Posted by sn...@apache.org.
USERGRID-1044: fixes
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/83d7a6f8
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/83d7a6f8
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/83d7a6f8
Branch: refs/heads/asf-site
Commit: 83d7a6f83e005ee8a4794ed86f4f0de95653360c
Parents: be0033e
Author: Mike Dunker <md...@apigee.com>
Authored: Tue Oct 27 13:44:33 2015 -0700
Committer: Mike Dunker <md...@apigee.com>
Committed: Tue Oct 27 13:44:33 2015 -0700
----------------------------------------------------------------------
.../usergrid/datagenerators/FeederGenerator.scala | 3 ++-
.../usergrid/scenarios/AuditScenarios.scala | 18 ++++++++----------
.../org/apache/usergrid/settings/Settings.scala | 10 +++++-----
3 files changed, 15 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/83d7a6f8/stack/loadtests/src/main/scala/org/apache/usergrid/datagenerators/FeederGenerator.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/datagenerators/FeederGenerator.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/datagenerators/FeederGenerator.scala
index 73b3d3c..38db19a 100755
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/datagenerators/FeederGenerator.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/datagenerators/FeederGenerator.scala
@@ -279,11 +279,12 @@ object FeederGenerator {
val name = if (line != null) array(1) else ""
val uuid = if (line != null) array(2) else ""
val modified = if (line != null) array(3) else ""
+ val lastStatus = if (line != null) array(4) else ""
//println(s"$collectionName|$name|$uuid|$modified")
val accessField = if (uuid != "") uuid else name
val queryField = if (uuid != "") s"uuid='$uuid'" else s"name='$name'"
- Map("collectionName" -> collectionName, "name" -> name, "uuid" -> uuid, "modified" -> modified,
+ Map("collectionName" -> collectionName, "name" -> name, "uuid" -> uuid, "modified" -> modified, "lastStatus" -> lastStatus,
"validEntity" -> validEntity, "accessField" -> accessField, "queryField" -> queryField)
}
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/83d7a6f8/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala
index 86c0b73..1ce2bc2 100644
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala
@@ -157,8 +157,7 @@ object AuditScenarios {
val modified = session("modified").as[String].toLong
val uuid = session("uuid").as[String]
val reqName = session("name").as[String]
- val prevStatus = session("status").as[Int]
- val prevError = session("error").as[String]
+ val lastStatus = session("lastStatus").as[String]
if (status == 200 || status == 404) {
val collectionEntities = session(SessionVarCollectionEntities).as[Seq[Any]]
val entityUuid = session(SessionVarEntityUuid).as[String]
@@ -166,12 +165,12 @@ object AuditScenarios {
val count = collectionEntities.length
if (count < 1) {
- Settings.addAuditUuid(uuid, collectionName, reqName, modified, status, s"NotFoundAtAll", prevStatus, prevError)
+ Settings.addAuditUuid(uuid, collectionName, reqName, modified, status, s"NotFoundAtAll", lastStatus)
Settings.incAuditNotFoundAtAll()
println(s"NOT FOUND AT ALL: $collectionName.$reqName ($uuid)")
} else if (count > 1) {
// invalid
- Settings.addAuditUuid(uuid, collectionName, reqName, modified, status, s"QueryInvalidCount$count", prevStatus, prevError)
+ Settings.addAuditUuid(uuid, collectionName, reqName, modified, status, s"QueryInvalidCount$count", lastStatus)
Settings.incAuditBadResponse()
println(s"INVALID RESPONSE (count=$count): $collectionName.$reqName ($uuid)")
} else {
@@ -199,13 +198,13 @@ object AuditScenarios {
println(s"PAYLOAD NAME MISMATCH (DIRECT): requestedName=$reqName returnedName=$entityName")
}
- Settings.addAuditUuid(uuid, collectionName, reqName, modified, status, errorString, prevStatus, prevError)
+ Settings.addAuditUuid(uuid, collectionName, reqName, modified, status, errorString, lastStatus)
Settings.incAuditNotFoundViaQuery()
println(s"NOT FOUND VIA QUERY: $collectionName.$reqName ($uuid)")
}
session
} else if (saveFailures) {
- Settings.addAuditUuid(uuid, collectionName, reqName, modified, status, "Failure", prevStatus, prevError)
+ Settings.addAuditUuid(uuid, collectionName, reqName, modified, status, "Failure", lastStatus)
session
} else {
session.markAsFailed
@@ -227,8 +226,7 @@ object AuditScenarios {
val uuid = session("uuid").as[String]
val reqName = session("name").as[String]
val modified = session("modified").as[String].toLong
- val prevStatus = session("status").as[Int]
- val prevError = session("error").as[String]
+ val lastStatus = session("lastStatus").as[String]
val collectionName = session(SessionVarCollectionName).as[String]
val entityUuid = session(SessionVarEntityUuid).as[String]
val entityName = session(SessionVarEntityName).as[String]
@@ -236,7 +234,7 @@ object AuditScenarios {
if (count < 1) {
// will check to see whether accessible directly
} else if (count > 1) {
- Settings.addAuditUuid(uuid, collectionName, reqName, modified, status, s"QueryInvalidCount$count", prevStatus, prevError)
+ Settings.addAuditUuid(uuid, collectionName, reqName, modified, status, s"QueryInvalidCount$count", lastStatus)
Settings.incAuditBadResponse()
println(s"INVALID RESPONSE (count=$count): $collectionName.$reqName ($uuid)")
} else {
@@ -268,7 +266,7 @@ object AuditScenarios {
// log even if technically successful -- we need to capture incorrect response
if (errorString != errorPrefix) {
- Settings.addAuditUuid(uuid, collectionName, reqName, modified, status, errorString, prevStatus, prevError)
+ Settings.addAuditUuid(uuid, collectionName, reqName, modified, status, errorString, lastStatus)
}
Settings.incAuditSuccess()
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/83d7a6f8/stack/loadtests/src/main/scala/org/apache/usergrid/settings/Settings.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/settings/Settings.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/settings/Settings.scala
index e0a0ec9..e27903a 100755
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/settings/Settings.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/settings/Settings.scala
@@ -291,10 +291,10 @@ object Settings {
val purgeUsers:Int = initIntSetting(ConfigProperties.PurgeUsers)
val uuidsHeader = "collection,name,uuid,modified,status"
- val uuidsFailHeader = "collection,name,uuid,modified,status,error,prevStatus,prevError"
+ val uuidsFailHeader = "collection,name,uuid,modified,status,error,lastStatus"
case class AuditList(var collection: String, var entityName: String, var uuid: String, var modified: Long, var status: Int)
case class AuditFailList(var collection: String, var entityName: String, var uuid: String, var modified: Long,
- var status: Int, var error: String, var prevStatus: Int, var prevError: String)
+ var status: Int, var error: String, var lastStatus: String)
//private var uuidMap: Map[Int, String] = Map()
private var uuidList: mutable.MutableList[AuditList] = mutable.MutableList[AuditList]()
@@ -383,10 +383,10 @@ object Settings {
private var lastAuditEntityCountPrinted: Long = 0L
private var auditUuidList: mutable.MutableList[AuditFailList] = mutable.MutableList[AuditFailList]()
def addAuditUuid(uuid: String, collection: String, entityName: String, modified: Long, status: Int, error: String,
- prevStatus: Int, prevError: String): Unit = {
+ lastStatus: String): Unit = {
if (captureAuditUuids) {
auditUuidList.synchronized {
- auditUuidList += AuditFailList(collection, entityName, uuid, modified, status, error, prevStatus, prevError)
+ auditUuidList += AuditFailList(collection, entityName, uuid, modified, status, error, lastStatus)
auditEntityCounter += 1L
if (logEntityProgress && (auditEntityCounter >= lastAuditEntityCountPrinted + entityProgressCount)) {
println(s"Entity: $auditEntityCounter")
@@ -406,7 +406,7 @@ object Settings {
writer.println(uuidsFailHeader)
val uuidList: List[AuditFailList] = auditUuidList.toList.sortBy(e => (e.collection, e.entityName, e.modified, e.status))
uuidList.foreach { e =>
- writer.println(s"${e.collection},${e.entityName},${e.uuid},${e.modified},${e.status},${e.error},${e.prevStatus},${e.prevError}")
+ writer.println(s"${e.collection},${e.entityName},${e.uuid},${e.modified},${e.status},${e.error},${e.lastStatus}")
}
writer.flush()
writer.close()
[03/50] [abbrv] usergrid git commit: Moves system endpoints into the
system package
Posted by sn...@apache.org.
http://git-wip-us.apache.org/repos/asf/usergrid/blob/7a4f36ab/stack/rest/src/main/java/org/apache/usergrid/rest/system/MigrateResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/system/MigrateResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/system/MigrateResource.java
new file mode 100644
index 0000000..c5a6dbc
--- /dev/null
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/system/MigrateResource.java
@@ -0,0 +1,270 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.usergrid.rest.system;
+
+
+import java.util.Map;
+import java.util.Set;
+
+import javax.ws.rs.*;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.UriInfo;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Scope;
+import org.springframework.stereotype.Component;
+
+import org.apache.usergrid.persistence.core.migration.data.DataMigrationManager;
+import org.apache.usergrid.persistence.core.migration.schema.MigrationManager;
+import org.apache.usergrid.rest.AbstractContextResource;
+import org.apache.usergrid.rest.ApiResponse;
+import org.apache.usergrid.rest.security.annotations.RequireSystemAccess;
+
+import com.fasterxml.jackson.databind.node.JsonNodeFactory;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.common.base.Preconditions;
+import com.google.inject.Injector;
+import com.sun.jersey.api.json.JSONWithPadding;
+
+
+@Component
+@Scope( "singleton" )
+@Produces( {
+ MediaType.APPLICATION_JSON,
+ "application/javascript",
+ "application/x-javascript",
+ "text/ecmascript",
+ "application/ecmascript",
+ "text/jscript"
+} )
+public class MigrateResource extends AbstractContextResource {
+
+ private static final Logger logger = LoggerFactory.getLogger( MigrateResource.class );
+
+ public MigrateResource() {
+ logger.info( "SystemResource initialized" );
+ }
+
+ @Autowired
+ private Injector guiceInjector;
+
+ @RequireSystemAccess
+ @PUT
+ @Path( "run" )
+ public JSONWithPadding migrateData( @Context UriInfo ui,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+ throws Exception {
+
+ ApiResponse response = createApiResponse();
+ response.setAction( "Migrate Data" );
+ //TODO make this use the task scheduler
+
+
+ final Thread migrate = new Thread() {
+
+ @Override
+ public void run() {
+
+ logger.info( "Migrating Schema" );
+
+ try {
+ getMigrationManager().migrate();
+ }
+ catch ( Exception e ) {
+ logger.error( "Unable to migrate data", e );
+ }
+
+ logger.info( "Migrating Data" );
+
+ try {
+ getDataMigrationManager().migrate();
+ }
+ catch ( Exception e ) {
+ logger.error( "Unable to migrate data", e );
+ }
+ }
+ };
+
+ migrate.setName( "Index migrate data formats" );
+ migrate.setDaemon( true );
+ migrate.start();
+
+ response.setSuccess();
+
+ return new JSONWithPadding( response, callback );
+ }
+
+ @RequireSystemAccess
+ @PUT
+ @Path( "run/{pluginName}" )
+ public JSONWithPadding migrateData(@PathParam("pluginName") String pluginName , @Context UriInfo ui,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+ throws Exception {
+
+ if(!getDataMigrationManager().pluginExists(pluginName)){
+ throw new IllegalArgumentException("Plugin doesn't exits name:"+pluginName);
+ }
+
+ ApiResponse response = createApiResponse();
+ response.setAction( "Migrate Data: "+ pluginName );
+ //TODO make this use the task scheduler
+
+
+
+
+ final Thread migrate = new Thread() {
+
+ @Override
+ public void run() {
+
+ logger.info( "Migrating Data for plugin: " + pluginName );
+
+ try {
+ getDataMigrationManager().migrate(pluginName);
+ }
+ catch ( Exception e ) {
+ logger.error( "Unable to migrate data for plugin: " + pluginName, e );
+ }
+ }
+ };
+
+ migrate.setName( "Index migrate data formats: "+pluginName );
+ migrate.setDaemon( true );
+ migrate.start();
+
+ response.setSuccess();
+
+ return new JSONWithPadding( response, callback );
+ }
+
+ @RequireSystemAccess
+ @PUT
+ @Path( "set" )
+ public JSONWithPadding setMigrationVersion(
+ @Context UriInfo ui, Map<String, Object> json,
+ @QueryParam( "callback" ) @DefaultValue( "" ) String callback )
+ throws Exception {
+
+ logger.debug( "setMigrationVersion" );
+
+ Preconditions.checkNotNull( json, "You must provide a json body" );
+ Preconditions.checkArgument( json.keySet().size() > 0, "You must specify at least one module and version" );
+
+ ApiResponse response = createApiResponse();
+ response.setAction("Set Migration Versions");
+
+ ObjectNode node = JsonNodeFactory.instance.objectNode();
+
+ final DataMigrationManager dataMigrationManager = getDataMigrationManager();
+ final Set<String> plugins = dataMigrationManager.getPluginNames();
+
+ /**
+ * Set the migration version for the plugins specified
+ */
+ for ( final String key : json.keySet() ) {
+
+ int version = ( int ) json.get( key );
+
+ dataMigrationManager.resetToVersion(key, version);
+ }
+
+
+ /**
+ * Echo back a response of the current versions for all plugins
+ */
+ for(final String pluginName: plugins){
+ node.put(pluginName, dataMigrationManager.getCurrentVersion(pluginName));
+ }
+
+
+ response.setData( node );
+ response.setSuccess();
+
+ return new JSONWithPadding( response, callback );
+ }
+
+
+ @RequireSystemAccess
+ @GET
+ @Path( "status" )
+ public JSONWithPadding migrateStatus(
+ @Context UriInfo ui,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+ throws Exception {
+
+ ApiResponse response = createApiResponse();
+ response.setAction( "Migrate Schema indexes" );
+
+ ObjectNode node = JsonNodeFactory.instance.objectNode();
+
+
+
+ final DataMigrationManager dataMigrationManager = getDataMigrationManager();
+
+ final Set<String> plugins = dataMigrationManager.getPluginNames();
+
+ for(final String pluginName: plugins){
+ node.put( pluginName, dataMigrationManager.getCurrentVersion( pluginName ) );
+ }
+
+ response.setData( node );
+
+ response.setSuccess();
+
+ return new JSONWithPadding( response, callback );
+ }
+
+
+ @RequireSystemAccess
+ @GET
+ @Path( "count" )
+ public JSONWithPadding migrateCount(
+ @Context UriInfo ui,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+ throws Exception {
+
+ ApiResponse response = createApiResponse();
+ response.setAction( "Current entity count in system" );
+
+ response.setProperty( "count", emf.performEntityCount() );
+
+ response.setSuccess();
+
+ return new JSONWithPadding( response, callback );
+ }
+
+
+ /**
+ * Get the schema migration manager
+ */
+ private MigrationManager getMigrationManager() {
+ return guiceInjector.getInstance( MigrationManager.class );
+ }
+
+ /**
+ * Get the Data migration manager
+ */
+ private DataMigrationManager getDataMigrationManager() {
+ return guiceInjector.getInstance( DataMigrationManager.class );
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/usergrid/blob/7a4f36ab/stack/rest/src/main/java/org/apache/usergrid/rest/system/SystemResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/system/SystemResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/system/SystemResource.java
new file mode 100644
index 0000000..a5174e6
--- /dev/null
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/system/SystemResource.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.rest.system;
+
+
+import javax.ws.rs.DefaultValue;
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.UriInfo;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.context.annotation.Scope;
+import org.springframework.stereotype.Component;
+
+import org.apache.usergrid.rest.AbstractContextResource;
+import org.apache.usergrid.rest.ApiResponse;
+import org.apache.usergrid.rest.security.annotations.RequireSystemAccess;
+
+import com.sun.jersey.api.json.JSONWithPadding;
+
+
+@Path( "/system" )
+@Component
+@Scope( "singleton" )
+@Produces( {
+ MediaType.APPLICATION_JSON, "application/javascript", "application/x-javascript", "text/ecmascript",
+ "application/ecmascript", "text/jscript"
+} )
+public class SystemResource extends AbstractContextResource {
+
+ private static final Logger logger = LoggerFactory.getLogger( SystemResource.class );
+
+
+ public SystemResource() {
+ logger.info( "SystemResource initialized" );
+ }
+
+
+ @RequireSystemAccess
+ @GET
+ @Path( "superuser/setup" )
+ public JSONWithPadding getSetupSuperuser( @Context UriInfo ui,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+ throws Exception {
+
+ ApiResponse response = createApiResponse();
+ response.setAction( "superuser setup" );
+
+ logger.info( "Setting up Superuser" );
+
+ try {
+ management.provisionSuperuser();
+ }
+ catch ( Exception e ) {
+ logger.error( "Unable to complete superuser setup", e );
+ }
+
+ response.setSuccess();
+
+ return new JSONWithPadding( response, callback );
+ }
+
+
+
+ @Path( "migrate" )
+ public MigrateResource migrate() {
+ return getSubResource( MigrateResource.class );
+ }
+
+
+ @Path( "index" )
+ public IndexResource index() { return getSubResource( IndexResource.class ); }
+
+
+ @Path( "database" )
+ public DatabaseResource database() {
+ return getSubResource( DatabaseResource.class );
+ }
+
+ @Path( "applications" )
+ public ApplicationsResource applications() {
+ return getSubResource( ApplicationsResource.class );
+ }
+
+
+ @Path( "connection" )
+ public ConnectionResource connection() { return getSubResource( ConnectionResource.class ); }
+
+}
[41/50] [abbrv] usergrid git commit: USERGRID-1044: add audit delete
test (delete only, no verify)
Posted by sn...@apache.org.
USERGRID-1044: add audit delete test (delete only, no verify)
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/247f7960
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/247f7960
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/247f7960
Branch: refs/heads/asf-site
Commit: 247f79603d1775273a56274ce7810f1b8f1f5d29
Parents: 0fc1804
Author: Mike Dunker <md...@apigee.com>
Authored: Mon Nov 2 07:34:42 2015 -0800
Committer: Mike Dunker <md...@apigee.com>
Committed: Mon Nov 2 07:34:42 2015 -0800
----------------------------------------------------------------------
stack/loadtests/runAuditDeleteEntities.sh | 65 ++++++++++++++++++++
.../runAuditVerifyCollectionEntities.sh | 2 +-
.../apache/usergrid/enums/ScenarioType.scala | 3 +-
.../usergrid/scenarios/AuditScenarios.scala | 49 ++++++++++++++-
.../usergrid/simulations/AuditSimulation.scala | 4 +-
5 files changed, 118 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/247f7960/stack/loadtests/runAuditDeleteEntities.sh
----------------------------------------------------------------------
diff --git a/stack/loadtests/runAuditDeleteEntities.sh b/stack/loadtests/runAuditDeleteEntities.sh
new file mode 100755
index 0000000..ff92940
--- /dev/null
+++ b/stack/loadtests/runAuditDeleteEntities.sh
@@ -0,0 +1,65 @@
+#!/bin/bash
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+DIR="${BASH_SOURCE%/*}"
+if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi
+. "$DIR/testConfig.sh"
+
+# from testConfig.sh
+#URL=
+#ADMIN_USER=
+#ADMIN_PASSWORD=
+#ORG=
+#APP=
+#AUTH_TYPE=
+#TOKEN_TYPE=
+#RETRY_COUNT=
+
+die() { echo "$@" 1>&2 ; exit 1; }
+
+[ "$#" -ge 3 ] || die "At least 3 arguments required, $# provided. Example is $0 RAMP_USERS RAMP_TIME(seconds) AUDIT_UUID_FILENAME [FAILED_UUID_FILENAME [USERGRID_REGION]]"
+
+RAMP_USERS="$1"
+RAMP_TIME="$2"
+AUDIT_UUID_FILENAME="$3"
+FAILED_UUID_FILENAME="$4"
+[ "$#" -ge 5 ] && USERGRID_REGION="$5"
+
+shift $#
+
+SCENARIO_TYPE=auditDeleteEntities
+
+#Compile everything
+mvn compile
+
+#Execute the test
+mvn gatling:execute \
+-DbaseUrl=${URL} \
+-DadminUser=${ADMIN_USER} \
+-DadminPassword=${ADMIN_PASSWORD} \
+-Dorg=${ORG} \
+-Dapp=${APP} \
+-DauthType=${AUTH_TYPE} \
+-DtokenType=${TOKEN_TYPE} \
+-DretryCount=${RETRY_COUNT} \
+-DscenarioType=${SCENARIO_TYPE} \
+-DrampUsers=${RAMP_USERS} \
+-DrampTime=${RAMP_TIME} \
+-DauditUuidFilename=${AUDIT_UUID_FILENAME} \
+-DfailedUuidFilename=${FAILED_UUID_FILENAME} \
+-DprintFailedRequests=${PRINT_FAILED_REQUESTS} \
+-DusergridRegion=${USERGRID_REGION} \
+-Dgatling.simulationClass=org.apache.usergrid.simulations.AuditSimulation
+
http://git-wip-us.apache.org/repos/asf/usergrid/blob/247f7960/stack/loadtests/runAuditVerifyCollectionEntities.sh
----------------------------------------------------------------------
diff --git a/stack/loadtests/runAuditVerifyCollectionEntities.sh b/stack/loadtests/runAuditVerifyCollectionEntities.sh
index 695c9ee..be50189 100755
--- a/stack/loadtests/runAuditVerifyCollectionEntities.sh
+++ b/stack/loadtests/runAuditVerifyCollectionEntities.sh
@@ -31,7 +31,7 @@ DELETE_AFTER_SUCCESSFUL_AUDIT=false
die() { echo "$@" 1>&2 ; exit 1; }
-[ "$#" -ge 3 ] || die "At least 3 arguments required, $# provided. Example is $0 RAMP_USERS RAMP_TIME(seconds) AUDIT_UUID_FILENAME [FAILED_UUID_FILENAME [DELETE_AFTER_SUCCESSFUL_AUDIT(true/false)[USERGRID_REGION]]]"
+[ "$#" -ge 3 ] || die "At least 3 arguments required, $# provided. Example is $0 RAMP_USERS RAMP_TIME(seconds) AUDIT_UUID_FILENAME [FAILED_UUID_FILENAME [DELETE_AFTER_SUCCESSFUL_AUDIT(true/false) [USERGRID_REGION]]]"
RAMP_USERS="$1"
RAMP_TIME="$2"
http://git-wip-us.apache.org/repos/asf/usergrid/blob/247f7960/stack/loadtests/src/main/scala/org/apache/usergrid/enums/ScenarioType.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/enums/ScenarioType.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/enums/ScenarioType.scala
index c86dc33..1cc1fa2 100644
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/enums/ScenarioType.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/enums/ScenarioType.scala
@@ -26,10 +26,11 @@ object ScenarioType {
val GetByNameSequential = "getByNameSequential"
val AuditGetCollectionEntities = "auditGetCollectionEntities"
val AuditVerifyCollectionEntities = "auditVerifyCollectionEntities"
+ val AuditDeleteEntities = "auditDeleteEntities"
val DoNothing = "doNothing"
val Values = Seq(GetAllByCursor,NameRandomInfinite,LoadEntities,DeleteEntities,UpdateEntities,UuidRandomInfinite,
- GetByNameSequential,AuditGetCollectionEntities,AuditVerifyCollectionEntities,DoNothing)
+ GetByNameSequential,AuditGetCollectionEntities,AuditVerifyCollectionEntities,AuditDeleteEntities,DoNothing)
def isValid(str: String): Boolean = {
Values.contains(str)
http://git-wip-us.apache.org/repos/asf/usergrid/blob/247f7960/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala
index cc59e0a..9ca479a 100644
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala
@@ -147,7 +147,6 @@ object AuditScenarios {
.get("/${collectionName}/${accessField}")
.headers(Headers.authToken)
.headers(Headers.usergridRegionHeaders)
- .check()
.check(status.in(Seq(200,404)),status.saveAs(SessionVarStatus),extractAuditEntities(SessionVarCollectionEntities),
extractEntityUuid(SessionVarEntityUuid),extractEntityName(SessionVarEntityName)))
.exec(session => {
@@ -282,7 +281,7 @@ object AuditScenarios {
// }
}
- val verifyCollections = scenario("Verify collections")
+ val verifyAuditedEntities = scenario("Verify audited entities")
.exec(injectTokenIntoSession())
.exec(injectAuthType())
.asLongAs(session => session("validEntity").asOption[String].map(validEntity => validEntity != "no").getOrElse[Boolean](true)) {
@@ -294,4 +293,50 @@ object AuditScenarios {
}
}
+ val deleteEntity = exec(
+ http("DELETE entity")
+ .delete("/${collectionName}/${accessField}")
+ .headers(Headers.authToken)
+ .headers(Headers.usergridRegionHeaders)
+ .check(status.saveAs(SessionVarStatus)))
+ .exec(session => {
+ val saveFailures = Settings.saveInvalidResponse
+ val status = session(SessionVarStatus).as[Int]
+ val collectionName = session(SessionVarCollectionName).as[String]
+ val modified = session("modified").as[String].toLong
+ val uuid = session("uuid").as[String]
+ val reqName = session("name").as[String]
+ val lastStatus = session("lastStatus").as[String]
+ if (status == 200) {
+ // success
+ Settings.incAuditSuccess()
+ session
+ } else if (status == 401 || status == 404) {
+ // didn't exist (currently returns 401, but 404 would be more appropriate)
+ Settings.addAuditUuid(uuid, collectionName, reqName, modified, status, s"NotFound", lastStatus)
+ Settings.incAuditEntryDeleteFailure()
+ session
+ } else if (saveFailures) {
+ // a different error
+ Settings.addAuditUuid(uuid, collectionName, reqName, modified, status, s"Error", lastStatus)
+ Settings.incAuditEntryDeleteFailure()
+ println(s"DELETE ERROR (status=$status): $collectionName.$reqName ($uuid)")
+ session.markAsFailed
+ } else {
+ session.markAsFailed
+ }
+
+ })
+
+ val deleteAuditedEntities = scenario("Delete audited entities")
+ .exec(injectTokenIntoSession())
+ .exec(injectAuthType())
+ .asLongAs(session => session("validEntity").asOption[String].map(validEntity => validEntity != "no").getOrElse[Boolean](true)) {
+ feed(FeederGenerator.collectionCsvFeeder)
+ .doIf(session => session("validEntity").as[String] == "yes") {
+ tryMax(if (Settings.saveInvalidResponse) 1 else 1+Settings.retryCount) {
+ exec(deleteEntity)
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/247f7960/stack/loadtests/src/main/scala/org/apache/usergrid/simulations/AuditSimulation.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/simulations/AuditSimulation.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/simulations/AuditSimulation.scala
index bdec3dd..ac900fd 100755
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/simulations/AuditSimulation.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/simulations/AuditSimulation.scala
@@ -33,7 +33,8 @@ class AuditSimulation extends Simulation {
def getScenario(scenarioType: String): ScenarioBuilder = {
scenarioType match {
case ScenarioType.AuditGetCollectionEntities => AuditScenarios.getAllCollections
- case ScenarioType.AuditVerifyCollectionEntities => AuditScenarios.verifyCollections
+ case ScenarioType.AuditVerifyCollectionEntities => AuditScenarios.verifyAuditedEntities
+ case ScenarioType.AuditDeleteEntities => AuditScenarios.deleteAuditedEntities
}
}
@@ -66,6 +67,7 @@ class AuditSimulation extends Simulation {
val uuidDesc = Settings.scenarioType match {
case ScenarioType.AuditGetCollectionEntities => "found"
case ScenarioType.AuditVerifyCollectionEntities => "failed"
+ case ScenarioType.AuditDeleteEntities => "failed"
}
Settings.writeAuditUuidsToFile(uuidDesc)
}
[24/50] [abbrv] usergrid git commit: Adds the ability to PUT
credentials as a superuser
Posted by sn...@apache.org.
Adds the ability to PUT credentials as a superuser
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/5ed8c7ce
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/5ed8c7ce
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/5ed8c7ce
Branch: refs/heads/asf-site
Commit: 5ed8c7ce14d253ba04dfad48d239a0e11bf1a33c
Parents: c652171
Author: Todd Nine <tn...@apigee.com>
Authored: Thu Oct 29 12:36:21 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Thu Oct 29 13:47:15 2015 -0600
----------------------------------------------------------------------
.../rest/applications/users/UserResource.java | 38 ++++++++++++++++++++
.../usergrid/management/ManagementService.java | 12 ++++++-
.../cassandra/ManagementServiceImpl.java | 21 ++++++++++-
3 files changed, 69 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5ed8c7ce/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java
index a8b0f81..df88cf0 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java
@@ -43,6 +43,7 @@ import org.slf4j.LoggerFactory;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component;
import org.apache.usergrid.management.ActivationState;
+import org.apache.usergrid.persistence.CredentialsInfo;
import org.apache.usergrid.persistence.EntityManager;
import org.apache.usergrid.persistence.index.query.Identifier;
import org.apache.usergrid.persistence.entities.User;
@@ -165,6 +166,43 @@ public class UserResource extends ServiceResource {
}
+ @PUT
+ @Path("credentials")
+ public JSONWithPadding setUserCredentials( @Context UriInfo ui, Map<String, Object> json,
+ @QueryParam("callback") @DefaultValue("callback") String callback )
+ throws Exception {
+
+ logger.info( "UserResource.setUserPassword" );
+
+ if ( json == null ) {
+ return null;
+ }
+
+ ApiResponse response = createApiResponse();
+ response.setAction( "set user credentials" );
+ Object credentials = json.get( "credentials" );
+
+
+ if ( credentials == null ) {
+ throw new IllegalArgumentException( "credentials sub object is required" );
+ }
+
+ UUID applicationId = getApplicationId();
+ UUID targetUserId = getUserUuid();
+
+ if ( targetUserId == null ) {
+ response.setError( "User not found" );
+ return new JSONWithPadding( response, callback );
+ }
+
+
+ management.setAppUserCredentialsInfo( applicationId, targetUserId, ( CredentialsInfo ) credentials );
+
+
+ return new JSONWithPadding( response, callback );
+ }
+
+
@POST
@Path("password")
public JSONWithPadding setUserPasswordPost( @Context UriInfo ui, Map<String, Object> json,
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5ed8c7ce/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java b/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java
index 3c5bbdb..d69de2e 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java
@@ -288,6 +288,16 @@ public interface ManagementService {
public void setAppUserPassword( UUID applicationId, UUID userId, String oldPassword, String newPassword )
throws Exception;
+ /**
+ * Set the credentials info into the
+ * @param applicationId
+ * @param userId
+ * @param credentialsInfo
+ * @throws Exception
+ */
+ void setAppUserCredentialsInfo( final UUID applicationId, final UUID userId, final CredentialsInfo credentialsInfo ) throws Exception;
+
+
public User verifyAppUserPasswordCredentials( UUID applicationId, String name, String password ) throws Exception;
public UserInfo getAppUserFromAccessToken( String token ) throws Exception;
@@ -350,7 +360,7 @@ public interface ManagementService {
public OrganizationConfig getOrganizationConfigForApplication( UUID applicationId ) throws Exception;
public void updateOrganizationConfig( OrganizationConfig organizationConfig ) throws Exception;
-
+
/**
* will delete all entities
* @param applicationId
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5ed8c7ce/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
index 70d74fc..2e33539 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
@@ -18,6 +18,7 @@ package org.apache.usergrid.management.cassandra;
import com.google.common.base.Optional;
+import com.google.common.base.Preconditions;
import com.google.common.collect.BiMap;
import com.google.common.collect.HashBiMap;
import com.google.inject.Injector;
@@ -2806,9 +2807,27 @@ public class ManagementServiceImpl implements ManagementService {
@Override
+ public void setAppUserCredentialsInfo( final UUID applicationId, final UUID userId,
+ final CredentialsInfo credentialsInfo ) throws Exception {
+
+ Preconditions.checkNotNull( applicationId, "applicationId is required" );
+ Preconditions.checkNotNull( userId, "userId is required" );
+ Preconditions.checkNotNull( credentialsInfo, "credentialsInfo is required" );
+
+ final User user = emf.getEntityManager( applicationId ).get(userId, User.class);
+
+ if(user == null){
+ throw new EntityNotFoundException( "User with id " + userId + " cannot be found" );
+ }
+
+ writeUserPassword(applicationId, user, credentialsInfo);
+ }
+
+
+ @Override
public User verifyAppUserPasswordCredentials( UUID applicationId, String name, String password ) throws Exception {
- User user = findUserEntity(applicationId, name);
+ User user = findUserEntity( applicationId, name );
if ( user == null ) {
return null;
}
[15/50] [abbrv] usergrid git commit: USERGRID-1044: print non-200s
during load (including body)
Posted by sn...@apache.org.
USERGRID-1044: print non-200s during load (including body)
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/03897447
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/03897447
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/03897447
Branch: refs/heads/asf-site
Commit: 0389744752f479d215f84ebbd5f25aa36d2108a4
Parents: 83d7a6f
Author: Mike Dunker <md...@apigee.com>
Authored: Tue Oct 27 14:59:10 2015 -0700
Committer: Mike Dunker <md...@apigee.com>
Committed: Tue Oct 27 14:59:10 2015 -0700
----------------------------------------------------------------------
.../usergrid/scenarios/EntityCollectionScenarios.scala | 8 +++++++-
1 file changed, 7 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/03897447/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
index f37a2d2..d5b3101 100644
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
@@ -42,6 +42,7 @@ object EntityCollectionScenarios {
val SessionVarStatus: String = "status"
val SessionVarUuid: String = "createUuid"
val SessionVarModified: String = "createModified"
+ val SessionVarBodyString: String = "bodyString"
def entityGetUrl(useCursor: Boolean): String = {
val url = s"/${Settings.collection}?" +
@@ -225,11 +226,12 @@ object EntityCollectionScenarios {
new ResponseWrapper(response) {
val contentType = response.header("content-type").getOrElse("").toLowerCase
val bodyStr = if (contentType.contains("json")) response.body.string else "[]"
+ if (bodyStr == "[]") { println(">>>>>>>>>>> USING EMPTY BODY") }
override val body = StringResponseBody(bodyStr, response.charset)
}
}
// 200 for success, 400 if already exists
- .check(status.saveAs(SessionVarStatus), extractEntityUuid(SessionVarUuid), extractEntityModified(SessionVarModified)))
+ .check(status.saveAs(SessionVarStatus), bodyString.saveAs(SessionVarBodyString), extractEntityUuid(SessionVarUuid), extractEntityModified(SessionVarModified)))
.exec(session => {
val saveFailures = Settings.saveInvalidResponse
val status = session(SessionVarStatus).as[Int]
@@ -239,6 +241,10 @@ object EntityCollectionScenarios {
val entityName = session("entityName").as[String]
val modified = if (status == 200) session(SessionVarModified).as[Long] else 0
val collectionName = session("collectionName").as[String]
+ if (status != 200) {
+ val bodyString = session(SessionVarBodyString).as[String]
+ println(s">>>>>>>> LOAD ERROR - Status: $status\nBody:\n$bodyString")
+ }
Settings.addUuid(uuid, collectionName, entityName, modified, status)
session
} else {
[10/50] [abbrv] usergrid git commit: USERGRID-1044: save previous
status and error in audit failure CSV
Posted by sn...@apache.org.
USERGRID-1044: save previous status and error in audit failure CSV
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/be0033ea
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/be0033ea
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/be0033ea
Branch: refs/heads/asf-site
Commit: be0033eac3f715f63ec716a43ee4d28bffed33dc
Parents: 9158772
Author: Mike Dunker <md...@apigee.com>
Authored: Tue Oct 27 12:56:05 2015 -0700
Committer: Mike Dunker <md...@apigee.com>
Committed: Tue Oct 27 12:56:05 2015 -0700
----------------------------------------------------------------------
.../usergrid/scenarios/AuditScenarios.scala | 18 +++++++++++-------
.../org/apache/usergrid/settings/Settings.scala | 12 +++++++-----
2 files changed, 18 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/be0033ea/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala
index 7a93eb6..86c0b73 100644
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala
@@ -32,7 +32,7 @@ object AuditScenarios {
//The value for the cursor
val SessionVarCursor: String = "cursor"
- val SessionVarStatus: String = "status"
+ val SessionVarStatus: String = "newStatus"
val SessionVarEntityUuid: String = "entityUuid"
val SessionVarEntityName: String = "entityName"
val SessionVarDeletedUuid: String = "deletedUuid"
@@ -157,6 +157,8 @@ object AuditScenarios {
val modified = session("modified").as[String].toLong
val uuid = session("uuid").as[String]
val reqName = session("name").as[String]
+ val prevStatus = session("status").as[Int]
+ val prevError = session("error").as[String]
if (status == 200 || status == 404) {
val collectionEntities = session(SessionVarCollectionEntities).as[Seq[Any]]
val entityUuid = session(SessionVarEntityUuid).as[String]
@@ -164,12 +166,12 @@ object AuditScenarios {
val count = collectionEntities.length
if (count < 1) {
- Settings.addAuditUuid(uuid, collectionName, reqName, modified, status, s"NotFoundAtAll")
+ Settings.addAuditUuid(uuid, collectionName, reqName, modified, status, s"NotFoundAtAll", prevStatus, prevError)
Settings.incAuditNotFoundAtAll()
println(s"NOT FOUND AT ALL: $collectionName.$reqName ($uuid)")
} else if (count > 1) {
// invalid
- Settings.addAuditUuid(uuid, collectionName, reqName, modified, status, s"QueryInvalidCount$count")
+ Settings.addAuditUuid(uuid, collectionName, reqName, modified, status, s"QueryInvalidCount$count", prevStatus, prevError)
Settings.incAuditBadResponse()
println(s"INVALID RESPONSE (count=$count): $collectionName.$reqName ($uuid)")
} else {
@@ -197,13 +199,13 @@ object AuditScenarios {
println(s"PAYLOAD NAME MISMATCH (DIRECT): requestedName=$reqName returnedName=$entityName")
}
- Settings.addAuditUuid(uuid, collectionName, reqName, modified, status, errorString)
+ Settings.addAuditUuid(uuid, collectionName, reqName, modified, status, errorString, prevStatus, prevError)
Settings.incAuditNotFoundViaQuery()
println(s"NOT FOUND VIA QUERY: $collectionName.$reqName ($uuid)")
}
session
} else if (saveFailures) {
- Settings.addAuditUuid(uuid, collectionName, reqName, modified, status, "Failure")
+ Settings.addAuditUuid(uuid, collectionName, reqName, modified, status, "Failure", prevStatus, prevError)
session
} else {
session.markAsFailed
@@ -225,6 +227,8 @@ object AuditScenarios {
val uuid = session("uuid").as[String]
val reqName = session("name").as[String]
val modified = session("modified").as[String].toLong
+ val prevStatus = session("status").as[Int]
+ val prevError = session("error").as[String]
val collectionName = session(SessionVarCollectionName).as[String]
val entityUuid = session(SessionVarEntityUuid).as[String]
val entityName = session(SessionVarEntityName).as[String]
@@ -232,7 +236,7 @@ object AuditScenarios {
if (count < 1) {
// will check to see whether accessible directly
} else if (count > 1) {
- Settings.addAuditUuid(uuid, collectionName, reqName, modified, status, s"QueryInvalidCount$count")
+ Settings.addAuditUuid(uuid, collectionName, reqName, modified, status, s"QueryInvalidCount$count", prevStatus, prevError)
Settings.incAuditBadResponse()
println(s"INVALID RESPONSE (count=$count): $collectionName.$reqName ($uuid)")
} else {
@@ -264,7 +268,7 @@ object AuditScenarios {
// log even if technically successful -- we need to capture incorrect response
if (errorString != errorPrefix) {
- Settings.addAuditUuid(uuid, collectionName, reqName, modified, status, errorString)
+ Settings.addAuditUuid(uuid, collectionName, reqName, modified, status, errorString, prevStatus, prevError)
}
Settings.incAuditSuccess()
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/be0033ea/stack/loadtests/src/main/scala/org/apache/usergrid/settings/Settings.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/settings/Settings.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/settings/Settings.scala
index 1db0a55..e0a0ec9 100755
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/settings/Settings.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/settings/Settings.scala
@@ -291,9 +291,10 @@ object Settings {
val purgeUsers:Int = initIntSetting(ConfigProperties.PurgeUsers)
val uuidsHeader = "collection,name,uuid,modified,status"
- val uuidsFailHeader = "collection,name,uuid,modified,status,error"
+ val uuidsFailHeader = "collection,name,uuid,modified,status,error,prevStatus,prevError"
case class AuditList(var collection: String, var entityName: String, var uuid: String, var modified: Long, var status: Int)
- case class AuditFailList(var collection: String, var entityName: String, var uuid: String, var modified: Long, var status: Int, var error: String)
+ case class AuditFailList(var collection: String, var entityName: String, var uuid: String, var modified: Long,
+ var status: Int, var error: String, var prevStatus: Int, var prevError: String)
//private var uuidMap: Map[Int, String] = Map()
private var uuidList: mutable.MutableList[AuditList] = mutable.MutableList[AuditList]()
@@ -381,10 +382,11 @@ object Settings {
private var auditEntityCounter: Long = 0L
private var lastAuditEntityCountPrinted: Long = 0L
private var auditUuidList: mutable.MutableList[AuditFailList] = mutable.MutableList[AuditFailList]()
- def addAuditUuid(uuid: String, collection: String, entityName: String, modified: Long, status: Int, error: String): Unit = {
+ def addAuditUuid(uuid: String, collection: String, entityName: String, modified: Long, status: Int, error: String,
+ prevStatus: Int, prevError: String): Unit = {
if (captureAuditUuids) {
auditUuidList.synchronized {
- auditUuidList += AuditFailList(collection, entityName, uuid, modified, status, error)
+ auditUuidList += AuditFailList(collection, entityName, uuid, modified, status, error, prevStatus, prevError)
auditEntityCounter += 1L
if (logEntityProgress && (auditEntityCounter >= lastAuditEntityCountPrinted + entityProgressCount)) {
println(s"Entity: $auditEntityCounter")
@@ -404,7 +406,7 @@ object Settings {
writer.println(uuidsFailHeader)
val uuidList: List[AuditFailList] = auditUuidList.toList.sortBy(e => (e.collection, e.entityName, e.modified, e.status))
uuidList.foreach { e =>
- writer.println(s"${e.collection},${e.entityName},${e.uuid},${e.modified},${e.status},${e.error}")
+ writer.println(s"${e.collection},${e.entityName},${e.uuid},${e.modified},${e.status},${e.error},${e.prevStatus},${e.prevError}")
}
writer.flush()
writer.close()
[38/50] [abbrv] usergrid git commit: Update the multi-tenant
migration script so we only have to use superuser.
Posted by sn...@apache.org.
Update the multi-tenant migration script so we only have to use superuser.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/eb2756b7
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/eb2756b7
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/eb2756b7
Branch: refs/heads/asf-site
Commit: eb2756b75ec06638cf50d5619b605fb312a62093
Parents: 40dfa06
Author: Michael Russo <mi...@gmail.com>
Authored: Fri Oct 30 13:18:32 2015 -0700
Committer: Michael Russo <mi...@gmail.com>
Committed: Fri Oct 30 13:18:32 2015 -0700
----------------------------------------------------------------------
stack/scripts/multitenant_migrate.py | 73 ++++++++++++-------------------
1 file changed, 27 insertions(+), 46 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/eb2756b7/stack/scripts/multitenant_migrate.py
----------------------------------------------------------------------
diff --git a/stack/scripts/multitenant_migrate.py b/stack/scripts/multitenant_migrate.py
index 5f1918e..d1131c1 100644
--- a/stack/scripts/multitenant_migrate.py
+++ b/stack/scripts/multitenant_migrate.py
@@ -20,9 +20,9 @@
#
# STEP 1 - SETUP TENANT ONE TOMCAT RUNNING 2.1 NOT IN SERVICE AND INIT MIGRATION
#
-# Login to the Tomcat instance and run this command, specifying both superuser and tenant organization admin creds:
+# Login to the Tomcat instance and run this command, specifying both superuser and tenant organization:
#
-# python migrate_entity_data.py --org <org1name> --super <user>:<pass> --admin <user>:<pass> --init
+# python migrate_entity_data.py --org <org1name> --super <user>:<pass> --init
#
# This command will setup and bootstrap the database, setup the migration system and update index mappings:
# - /system/database/setup
@@ -39,7 +39,7 @@
#
# On the same Tomcat instance and run this command with the --date timestamp you noted in the previous step:
#
-# python migrate_entity_data.py --org <org1name> --super <user>:<pass> --admin <user>:<pass> --date <timestamp>
+# python migrate_entity_data.py --org <org1name> --super <user>:<pass> --date <timestamp>
#
# Then it will migrate appinfos, re-index the management app and then for each of the specified org's apps
# it will de-dup connections and re-index the app with a start-date specified so only data modified since
@@ -47,9 +47,9 @@
#
# STEP 3 - SETUP TENANT TWO TOMCAT RUNNING 2.1 NOT IN SERVICE
#
-# Login to the Tomcat instance and run this command, specifying both superuser and tenant organization admin creds:
+# Login to the Tomcat instance and run this command, specifying both superuser and tenant organization:
#
-# python migrate_entity_data.py --org <org2name> --super <user>:<pass> --admin <user>:<pass>
+# python migrate_entity_data.py --org <org2name> --super <user>:<pass>
#
# This command will migrate appinfos, re-index the management app and then for each of the specified org's apps
# it will de-dup connections and re-index the app.
@@ -60,17 +60,17 @@
#
# On the same Tomcat instance and run this command with the --date timestamp you noted in the previous step:
#
-# python migrate_entity_data.py --org <org2name> --super <user>:<pass> --admin <user>:<pass> --date <timestamp>
+# python migrate_entity_data.py --org <org2name> --super <user>:<pass> --date <timestamp>
#
# Then it will migrate appinfos, re-index the management app and then for each of the specified org's apps
# it will de-dup connections and re-index the app with a start-date specified so only data modified since
# STEP 1 will be re-indexed.
#
-# STEP 5 - FULL DATA MIGRATION
+# STEP 5 - FULL DATA MIGRATION (migrates entity data to new format)
#
-# Login to any Tomcat instance in the cluster and run this command (admin user creds must be specificed but will be ignored):
+# Login to any Tomcat instance in the cluster and run this command:
#
-# python migrate_entity_data.py --super <user>:<pass> --admin <user>:<pass> --full
+# python migrate_entity_data.py --super <user>:<pass> --full
#
# This command will run the full data migration.
#
@@ -118,11 +118,6 @@ def parse_args():
type=str,
required=True)
- parser.add_argument('--admin',
- help='Organization admin creds <user:pass>',
- type=str,
- required=True)
-
parser.add_argument('--init',
help='Init system and start first migration.',
action='store_true',
@@ -154,14 +149,6 @@ def parse_args():
arg_vars['superuser'] = creds[0]
arg_vars['superpass'] = creds[1]
- creds = arg_vars['super'].split(':')
- if len(creds) != 2:
- print('Org admin credentials not properly specified. Must be "-u <user:pass>". Exiting...')
- exit_on_error()
- else:
- arg_vars['adminuser'] = creds[0]
- arg_vars['adminpass'] = creds[1]
-
return arg_vars
@@ -180,8 +167,6 @@ class Migrate:
self.logger = init_logging(self.__class__.__name__)
self.super_user = self.args['superuser']
self.super_pass = self.args['superpass']
- self.admin_user = self.args['adminuser']
- self.admin_pass = self.args['adminpass']
self.org = self.args['org']
self.init = self.args['init']
self.full = self.args['full']
@@ -220,7 +205,6 @@ class Migrate:
return
-
if self.init:
# Init the migration system as this is the first migration done on the cluster
@@ -250,7 +234,6 @@ class Migrate:
if index_mapping_updated:
break
-
# Migrate app info
if self.is_appinfo_migrated():
@@ -271,7 +254,6 @@ class Migrate:
break
self.logger.info('AppInfo Migration Ended.')
-
# Reindex management app
job = self.start_app_reindex(MANAGEMENT_APP_ID)
@@ -287,7 +269,6 @@ class Migrate:
self.logger.info("Finished Re-index. Job=[%s]", job)
self.metrics['reindex_end'] = get_current_time()
-
# Dedup and re-index all of organization's apps
app_ids = self.get_app_ids()
@@ -323,7 +304,6 @@ class Migrate:
self.log_metrics()
self.logger.info("Finished...")
-
except KeyboardInterrupt:
self.log_metrics()
self.logger.error('Keyboard interrupted migration. Please run again to ensure the migration finished.')
@@ -362,7 +342,7 @@ class Migrate:
def start_core_data_migration(self):
try:
- r = requests.put(url=self.get_migration_url(), auth=(self.admin_user, self.super_pass))
+ r = requests.put(url=self.get_migration_url(), auth=(self.super_user, self.super_pass))
response = r.json()
return response
except requests.exceptions.RequestException as e:
@@ -371,7 +351,7 @@ class Migrate:
def start_fulldata_migration(self):
try:
- r = requests.put(url=self.get_migration_url(), auth=(self.admin_user, self.super_pass))
+ r = requests.put(url=self.get_migration_url(), auth=(self.super_user, self.super_pass))
response = r.json()
return response
except requests.exceptions.RequestException as e:
@@ -382,7 +362,7 @@ class Migrate:
try:
# TODO fix this URL
migrateUrl = self.get_migration_url() + '/' + PLUGIN_MIGRATION_SYSTEM
- r = requests.put(url=migrateUrl, auth=(self.admin_user, self.super_pass))
+ r = requests.put(url=migrateUrl, auth=(self.super_user, self.super_pass))
response = r.json()
return response
except requests.exceptions.RequestException as e:
@@ -424,7 +404,7 @@ class Migrate:
def start_appinfo_migration(self):
try:
migrateUrl = self.get_migration_url() + '/' + PLUGIN_APPINFO
- r = requests.put(url=migrateUrl, auth=(self.admin_user, self.admin_pass))
+ r = requests.put(url=migrateUrl, auth=(self.super_user, self.super_pass))
response = r.json()
return response
except requests.exceptions.RequestException as e:
@@ -639,24 +619,25 @@ class Migrate:
try:
- url = self.endpoint + "/management/token"
- body = json.dumps({"grant_type":"password","username":self.admin_user,"password":self.admin_pass})
- r = requests.post(url=url, data=body)
- if ( r.status_code != 200 ):
- print "Error logging in: " + r.text
- return
-
- access_token = r.json()["access_token"]
+ url = self.endpoint + "/management/organizations"
+ r = requests.get(url=url, auth=(self.super_user, self.super_pass))
- url = self.endpoint + "/management/orgs/" + self.org + "/apps?access_token=" + access_token
- r = requests.get(url=url)
if r.status_code != 200:
exit_on_error('Cannot get app ids: ' + r.text)
- apps = r.json()["data"]
+ response_json = r.json()
+
app_ids = []
- for appId in apps.values():
- app_ids.append(appId)
+ orgs = response_json["organizations"]
+ if orgs is not None:
+ for org in orgs:
+ if org["name"] == self.org:
+ for app in org["applications"]:
+ app_ids.append(org["applications"][app])
+ else:
+ e = 'No Orgs in this system'
+ self.logger.error(e)
+ exit_on_error(e)
return app_ids
[47/50] [abbrv] usergrid git commit: This closes #312 - closed at
request of developer
Posted by sn...@apache.org.
This closes #312 - closed at request of developer
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/38a7d623
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/38a7d623
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/38a7d623
Branch: refs/heads/asf-site
Commit: 38a7d62364637eef53b7a184e0e66cc2f46eef55
Parents: 61a25bd
Author: Mike Dunker <md...@apigee.com>
Authored: Tue Nov 3 14:54:15 2015 -0800
Committer: Mike Dunker <md...@apigee.com>
Committed: Tue Nov 3 14:54:15 2015 -0800
----------------------------------------------------------------------
----------------------------------------------------------------------
[39/50] [abbrv] usergrid git commit: Merge branch '2.1-release' of
https://git-wip-us.apache.org/repos/asf/usergrid
Posted by sn...@apache.org.
Merge branch '2.1-release' of https://git-wip-us.apache.org/repos/asf/usergrid
# Conflicts:
# stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
# stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
# stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/InMemoryAsyncEventService.java
# stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java
# stack/rest/src/main/java/org/apache/usergrid/rest/applications/ApplicationResource.java
# stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java
# stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationsResource.java
# stack/rest/src/main/java/org/apache/usergrid/rest/system/MigrateResource.java
# stack/rest/src/main/java/org/apache/usergrid/rest/system/SystemResource.java
# stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/UserResourceIT.java
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/b422364e
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/b422364e
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/b422364e
Branch: refs/heads/asf-site
Commit: b422364e20c14a657de076b793bc6b7c1160a556
Parents: 9d3cf5b eb2756b
Author: Todd Nine <tn...@apigee.com>
Authored: Fri Oct 30 16:32:23 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Fri Oct 30 16:32:23 2015 -0600
----------------------------------------------------------------------
.../corepersistence/CpEntityManagerFactory.java | 7 +-
.../corepersistence/CpRelationManager.java | 16 +-
.../asyncevents/AmazonAsyncEventService.java | 99 +--
.../asyncevents/AsyncEventService.java | 5 +
.../asyncevents/EventBuilder.java | 25 +-
.../asyncevents/EventBuilderImpl.java | 60 +-
.../index/IndexProcessorFig.java | 15 +-
.../corepersistence/index/IndexServiceImpl.java | 5 +-
.../pipeline/read/FilterResult.java | 7 +
.../read/traverse/AbstractReadGraphFilter.java | 136 +++-
.../read/traverse/EdgeCursorSerializer.java | 8 +-
.../read/traverse/EntityLoadVerifyFilter.java | 24 +-
.../traverse/ReadGraphCollectionFilter.java | 11 +-
.../ReadGraphConnectionByTypeFilter.java | 11 +-
.../traverse/ReadGraphConnectionFilter.java | 11 +-
.../results/ObservableQueryExecutor.java | 7 +
.../service/StatusServiceImpl.java | 3 +-
.../usergrid/persistence/CredentialsInfo.java | 46 ++
.../persistence/ObservableIterator.java | 83 ---
.../index/AsyncIndexServiceTest.java | 3 +-
.../corepersistence/index/IndexServiceTest.java | 12 +-
.../pipeline/cursor/CursorTest.java | 24 +-
.../service/ConnectionServiceImplTest.java | 5 +-
.../persistence/ApplicationServiceIT.java | 4 +-
.../persistence/core/astyanax/CassandraFig.java | 2 +-
.../core/astyanax/MultiRowColumnIterator.java | 46 +-
.../persistence/core/rx/ObservableIterator.java | 2 +-
.../usergrid/persistence/graph/GraphFig.java | 4 +-
.../persistence/graph/GraphManager.java | 14 +-
.../usergrid/persistence/graph/MarkedEdge.java | 15 +-
.../persistence/graph/guice/GraphModule.java | 11 +-
.../graph/impl/GraphManagerImpl.java | 90 ++-
.../graph/impl/SimpleMarkedEdge.java | 62 +-
.../impl/stage/NodeDeleteListenerImpl.java | 27 +-
.../impl/EdgeSerializationImpl.java | 17 +-
.../impl/shard/AsyncTaskExecutor.java | 34 +
.../graph/serialization/impl/shard/Shard.java | 15 +
.../impl/shard/ShardEntryGroup.java | 13 +-
.../impl/shard/ShardGroupCompaction.java | 4 -
.../impl/shard/ShardGroupDeletion.java | 78 +++
.../impl/shard/impl/AsyncTaskExecutorImpl.java | 53 ++
.../shard/impl/NodeShardAllocationImpl.java | 81 ++-
.../shard/impl/ShardGroupColumnIterator.java | 72 +-
.../shard/impl/ShardGroupCompactionImpl.java | 10 +-
.../impl/shard/impl/ShardGroupDeletionImpl.java | 230 +++++++
.../impl/shard/impl/ShardsColumnIterator.java | 10 +
.../persistence/graph/GraphManagerIT.java | 76 +--
.../persistence/graph/GraphManagerLoadTest.java | 10 +-
.../graph/GraphManagerShardConsistencyIT.java | 382 ++++++++---
.../graph/GraphManagerStressTest.java | 16 +-
.../impl/shard/ShardEntryGroupTest.java | 14 +
.../impl/shard/ShardGroupCompactionTest.java | 30 +-
.../shard/impl/ShardGroupDeletionImplTest.java | 341 +++++++++
stack/corepersistence/pom.xml | 2 +-
.../persistence/index/CandidateResult.java | 11 +-
.../persistence/index/EntityIndexBatch.java | 2 +-
.../usergrid/persistence/index/IndexFig.java | 6 +
.../index/impl/DeIndexOperation.java | 5 +
.../index/impl/EsEntityIndexBatchImpl.java | 34 +-
.../index/impl/EsEntityIndexImpl.java | 188 +++--
.../persistence/index/impl/IndexingUtils.java | 2 +-
.../persistence/index/impl/EntityIndexTest.java | 14 +-
.../usergrid/persistence/queue/QueueFig.java | 2 +-
.../org/apache/usergrid/rest/IndexResource.java | 350 ----------
.../apache/usergrid/rest/MigrateResource.java | 275 --------
.../apache/usergrid/rest/SystemResource.java | 111 ---
.../rest/applications/ApplicationResource.java | 50 ++
.../rest/applications/users/UserResource.java | 134 +++-
.../organizations/OrganizationsResource.java | 33 +
.../rest/system/ConnectionResource.java | 216 ++++++
.../usergrid/rest/system/DatabaseResource.java | 2 +
.../usergrid/rest/system/IndexResource.java | 353 ++++++++++
.../usergrid/rest/system/MigrateResource.java | 277 ++++++++
.../usergrid/rest/system/SystemResource.java | 121 ++++
.../collection/users/UserResourceIT.java | 117 ++++
stack/scripts/create_test_data.py | 222 ++++++
stack/scripts/migrate_entity_data.py | 18 +-
stack/scripts/multitenant_migrate.py | 683 +++++++++++++++++++
.../management/AppInfoMigrationPlugin.java | 5 +-
.../usergrid/management/ManagementService.java | 14 +-
.../cassandra/ManagementServiceImpl.java | 43 +-
81 files changed, 4231 insertions(+), 1445 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/b422364e/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/b422364e/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/b422364e/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
index 14fbdaa,6b9abbc..e5c25fb
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
@@@ -315,25 -316,34 +315,31 @@@ public class AmazonAsyncEventService im
boolean validateEmptySets = true;
Observable<IndexOperationMessage> indexoperationObservable;
//merge each operation to a master observable;
- if (event instanceof EdgeDeleteEvent) {
- indexoperationObservable = handleEdgeDelete(message);
- } else if (event instanceof EdgeIndexEvent) {
- indexoperationObservable = handleEdgeIndex(message);
- } else if (event instanceof EntityDeleteEvent) {
- indexoperationObservable = handleEntityDelete(message);
- } else if (event instanceof EntityIndexEvent) {
- indexoperationObservable = handleEntityIndexUpdate(message);
- } else if (event instanceof InitializeApplicationIndexEvent) {
+ if ( event instanceof EdgeDeleteEvent ) {
+ indexoperationObservable = handleEdgeDelete( message );
+ }
+ else if ( event instanceof EdgeIndexEvent ) {
+ indexoperationObservable = handleEdgeIndex( message );
+ }
+ else if ( event instanceof EntityDeleteEvent ) {
+ indexoperationObservable = handleEntityDelete( message );
+ validateEmptySets = false; // do not check this one for an empty set b/c it can be empty
+
+ }
+ else if ( event instanceof EntityIndexEvent ) {
+ indexoperationObservable = handleEntityIndexUpdate( message );
+ }
+ else if ( event instanceof InitializeApplicationIndexEvent ) {
//does not return observable
- handleInitializeApplicationIndex( event, message );
- indexoperationObservable = Observable.just( new IndexOperationMessage() );
+ handleInitializeApplicationIndex(event, message);
+ indexoperationObservable = Observable.just(new IndexOperationMessage());
validateEmptySets = false; //do not check this one for an empty set b/c it will be empty.
- }
- else if ( event instanceof ElasticsearchIndexEvent ) {
- handleIndexOperation( ( ElasticsearchIndexEvent ) event );
- indexoperationObservable = Observable.just( new IndexOperationMessage() );
+ } else if (event instanceof ElasticsearchIndexEvent) {
+ handleIndexOperation((ElasticsearchIndexEvent) event);
+ indexoperationObservable = Observable.just(new IndexOperationMessage());
validateEmptySets = false; //do not check this one for an empty set b/c it will be empty.
- }
-
- else {
- throw new Exception( "Unknown EventType" );//TODO: print json instead
+ } else {
+ throw new Exception("Unknown EventType");//TODO: print json instead
}
//collect all of the
http://git-wip-us.apache.org/repos/asf/usergrid/blob/b422364e/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java
index 69d5e18,9d02717..c991b36
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java
@@@ -70,10 -72,19 +72,19 @@@ public interface IndexProcessorFig exte
int getIndexQueueVisibilityTimeout();
/**
+ * The number of worker threads used when handing off messages from the SQS thread
+ */
+ @Default( "20" )
+ @Key( EVENT_CONCURRENCY_FACTOR )
+ int getEventConcurrencyFactor();
+
+
+
+ /**
* The number of worker threads used to read index write requests from the queue.
*/
- @Default( "8" )
- @Key( ELASTICSEARCH_WORKER_COUNT )
+ @Default("16")
+ @Key(ELASTICSEARCH_WORKER_COUNT)
int getWorkerCount();
/**
http://git-wip-us.apache.org/repos/asf/usergrid/blob/b422364e/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
index 7efe8e4,b2a1a2a..301a7ae
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
@@@ -218,10 -219,10 +219,10 @@@ public class IndexServiceImpl implement
//collect results into a single batch
.collect( () -> ei.createBatch(), ( batch, candidateResult ) -> {
logger.debug( "Deindexing on edge {} for entity {} added to batch",searchEdge , entityId );
- batch.deindex( searchEdge, candidateResult );
+ batch.deindex( candidateResult );
} )
//return the future from the batch execution
- .flatMap( batch ->Observable.just(batch.build()) );
+ .map( batch ->batch.build() );
return ObservableTimer.time(batches, indexTimer);
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/b422364e/stack/core/src/test/java/org/apache/usergrid/persistence/ApplicationServiceIT.java
----------------------------------------------------------------------
diff --cc stack/core/src/test/java/org/apache/usergrid/persistence/ApplicationServiceIT.java
index f8079e5,658d3eb..9ad90eb
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/ApplicationServiceIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/ApplicationServiceIT.java
@@@ -85,15 -87,10 +86,16 @@@ public class ApplicationServiceIT exten
, Long.MAX_VALUE, SearchByEdgeType.Order.DESCENDING,
Optional.<Edge>absent() );
- Iterator<Edge> results = graphManager.loadEdgesFromSource(simpleSearchByEdgeType).toBlocking().getIterator();
+ Iterator<MarkedEdge>
+ results = graphManager.loadEdgesFromSource(simpleSearchByEdgeType).toBlocking().getIterator();
if(results.hasNext()){
- Assert.fail("should be empty");
+ int i = 0;
+
+ while(results.hasNext()){
+ results.next();
+ i++;
+ }
+ Assert.fail("should be empty but has "+i);
}else{
Results searchCollection = entityManager.searchCollection(entityManager.getApplication(), "tests", Query.all());
http://git-wip-us.apache.org/repos/asf/usergrid/blob/b422364e/stack/corepersistence/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/b422364e/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
index 1ffcd02,a8fb751..a49e217
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
@@@ -117,7 -136,13 +136,12 @@@ public class EsEntityIndexBatchImpl imp
return deindex( searchEdge, entity.getId(), entity.getVersion() );
}
+ @Override
+ public EntityIndexBatch deindex( final CandidateResult entity ) {
+
+ return deindexWithDocId(entity.getDocId());
+ }
-
@Override
public IndexOperationMessage build() {
return container;
http://git-wip-us.apache.org/repos/asf/usergrid/blob/b422364e/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/b422364e/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/b422364e/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ApplicationResource.java
----------------------------------------------------------------------
diff --cc stack/rest/src/main/java/org/apache/usergrid/rest/applications/ApplicationResource.java
index e830876,162565f..769b836
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ApplicationResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ApplicationResource.java
@@@ -17,7 -17,9 +17,8 @@@
package org.apache.usergrid.rest.applications;
+ import com.fasterxml.jackson.databind.ObjectMapper;
-import com.sun.jersey.api.json.JSONWithPadding;
-import com.sun.jersey.api.view.Viewable;
+import com.fasterxml.jackson.jaxrs.json.annotation.JSONP;
import org.apache.amber.oauth2.common.error.OAuthError;
import org.apache.amber.oauth2.common.exception.OAuthProblemException;
import org.apache.amber.oauth2.common.message.OAuthResponse;
@@@ -47,9 -48,9 +48,10 @@@ import org.apache.usergrid.rest.excepti
import org.apache.usergrid.rest.exceptions.RedirectionException;
import org.apache.usergrid.rest.security.annotations.RequireApplicationAccess;
import org.apache.usergrid.rest.security.annotations.RequireOrganizationAccess;
+ import org.apache.usergrid.rest.security.annotations.RequireSystemAccess;
import org.apache.usergrid.security.oauth.AccessInfo;
import org.apache.usergrid.security.oauth.ClientCredentialsInfo;
+import org.glassfish.jersey.server.mvc.Viewable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.context.annotation.Scope;
@@@ -601,6 -601,54 +603,54 @@@ public class ApplicationResource extend
if(value==null){
throw new EntityNotFoundException("apigeeMobileConfig not found, it is possibly not enabled for your config.");
}
- return new JSONWithPadding( value, callback );
+ return value;
}
+
+ // Specifically require superuser access as this is setting app properties directly (only way to currently do this
+ // with Apigee's apigeeMobileConfig
+ @RequireOrganizationAccess
+ @POST
+ @Path("apm/apigeeMobileConfig")
+ @Consumes(APPLICATION_JSON)
+ @Produces(MediaType.APPLICATION_JSON)
- public JSONWithPadding setAPMConfig( @Context UriInfo ui,
- @QueryParam("callback") @DefaultValue("callback") String callback,
- Map<String, Object> json) throws Exception {
++ public String setAPMConfig( @Context UriInfo ui,
++ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback,
++ Map<String, Object> json ) throws Exception {
+
+ if(json == null || json.size() < 1){
+ logger.error("Param {} cannot be null for POST apm/apigeeMobileConfig", APIGEE_MOBILE_APM_CONFIG_JSON_KEY);
+ throw new IllegalArgumentException("Request body cannot be empty and must include apigeeMobileConfig params");
+ }
+
+ final String requestAppUUID = (String) json.get("applicationUUID");
+ if(!requestAppUUID.equalsIgnoreCase(applicationId.toString())){
+ logger.error("Provided application UUID {} does not match actual application UUID {}",
+ requestAppUUID,
+ applicationId.toString());
+ throw new IllegalArgumentException(
+ String.format("Provided application UUID %s does not match actual application UUID %s",
+ requestAppUUID,
+ applicationId.toString())
+ );
+ }
+
+ final String apmConfig = new ObjectMapper().writeValueAsString(json);
+ if(logger.isDebugEnabled()){
+ logger.debug("Received request to set apigeeMobileConfig properties with: {}", apmConfig);
+ }
+
+
+ EntityManager em = emf.getEntityManager( applicationId );
+ em.setProperty(new SimpleEntityRef(Application.ENTITY_TYPE, applicationId),
+ APIGEE_MOBILE_APM_CONFIG_JSON_KEY,
+ apmConfig
+ );
+
+ logger.info("Successfully set apigeeMobileConfig properties with: {}", apmConfig);
+
- return new JSONWithPadding(apmConfig, callback);
++ return apmConfig;
+
+ }
+
+
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/b422364e/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java
----------------------------------------------------------------------
diff --cc stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java
index b84fc08,fb2962e..130658cd
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java
@@@ -17,38 -17,63 +17,63 @@@
package org.apache.usergrid.rest.applications.users;
--import com.fasterxml.jackson.databind.JsonNode;
--import com.fasterxml.jackson.databind.ObjectMapper;
- import com.fasterxml.jackson.jaxrs.json.annotation.JSONP;
- import net.tanesha.recaptcha.ReCaptchaImpl;
- import net.tanesha.recaptcha.ReCaptchaResponse;
+ import java.util.Map;
+ import java.util.UUID;
+
+ import javax.ws.rs.Consumes;
+ import javax.ws.rs.DefaultValue;
+ import javax.ws.rs.FormParam;
+ import javax.ws.rs.GET;
+ import javax.ws.rs.POST;
+ import javax.ws.rs.PUT;
+ import javax.ws.rs.Path;
+ import javax.ws.rs.PathParam;
+ import javax.ws.rs.Produces;
+ import javax.ws.rs.QueryParam;
+ import javax.ws.rs.core.Context;
+ import javax.ws.rs.core.MediaType;
+ import javax.ws.rs.core.PathSegment;
+ import javax.ws.rs.core.Response;
+ import javax.ws.rs.core.UriInfo;
+
++import org.glassfish.jersey.server.mvc.Viewable;
+ import org.slf4j.Logger;
+ import org.slf4j.LoggerFactory;
+ import org.springframework.context.annotation.Scope;
+ import org.springframework.stereotype.Component;
+
import org.apache.amber.oauth2.common.exception.OAuthProblemException;
import org.apache.amber.oauth2.common.message.OAuthResponse;
import org.apache.commons.lang.StringUtils;
+
import org.apache.usergrid.management.ActivationState;
+ import org.apache.usergrid.persistence.CredentialsInfo;
import org.apache.usergrid.persistence.EntityManager;
-import org.apache.usergrid.persistence.index.query.Identifier;
import org.apache.usergrid.persistence.entities.User;
+import org.apache.usergrid.persistence.index.query.Identifier;
import org.apache.usergrid.rest.AbstractContextResource;
import org.apache.usergrid.rest.ApiResponse;
import org.apache.usergrid.rest.applications.ServiceResource;
import org.apache.usergrid.rest.exceptions.RedirectionException;
import org.apache.usergrid.rest.security.annotations.RequireApplicationAccess;
+ import org.apache.usergrid.rest.security.annotations.RequireSystemAccess;
import org.apache.usergrid.security.oauth.AccessInfo;
import org.apache.usergrid.security.tokens.exceptions.TokenException;
- import org.glassfish.jersey.server.mvc.Viewable;
- import org.slf4j.Logger;
- import org.slf4j.LoggerFactory;
- import org.springframework.context.annotation.Scope;
- import org.springframework.stereotype.Component;
- import javax.ws.rs.*;
- import javax.ws.rs.core.*;
- import java.util.Map;
- import java.util.UUID;
-import com.sun.jersey.api.json.JSONWithPadding;
-import com.sun.jersey.api.view.Viewable;
++import com.fasterxml.jackson.databind.JsonNode;
++import com.fasterxml.jackson.databind.ObjectMapper;
++import com.fasterxml.jackson.jaxrs.json.annotation.JSONP;
- import static javax.servlet.http.HttpServletResponse.*;
- import static org.apache.usergrid.security.shiro.utils.SubjectUtils.*;
+ import net.tanesha.recaptcha.ReCaptchaImpl;
+ import net.tanesha.recaptcha.ReCaptchaResponse;
+
+ import static javax.servlet.http.HttpServletResponse.SC_BAD_REQUEST;
+ import static javax.servlet.http.HttpServletResponse.SC_FORBIDDEN;
+ import static javax.servlet.http.HttpServletResponse.SC_OK;
+
+ import static org.apache.usergrid.security.shiro.utils.SubjectUtils.getSubjectUserId;
+ import static org.apache.usergrid.security.shiro.utils.SubjectUtils.isApplicationAdmin;
+ import static org.apache.usergrid.security.shiro.utils.SubjectUtils.isApplicationUser;
import static org.apache.usergrid.utils.ConversionUtils.string;
@@@ -143,9 -164,84 +168,84 @@@ public class UserResource extends Servi
management.setAppUserPassword( getApplicationId(), targetUserId, oldPassword, newPassword );
}
- return new JSONWithPadding( response, callback );
+ return response;
}
+ @GET
+ @RequireSystemAccess
+ @Path("credentials")
- public JSONWithPadding getUserCredentials(@QueryParam("callback") @DefaultValue("callback") String callback )
++ public ApiResponse getUserCredentials(@QueryParam("callback") @DefaultValue("callback") String callback )
+ throws Exception {
+
+ logger.info( "UserResource.getUserCredentials" );
+
+
+ final ApiResponse response = createApiResponse();
+ response.setAction( "get user credentials" );
+
+ final UUID applicationId = getApplicationId();
+ final UUID targetUserId = getUserUuid();
+
+ if ( applicationId == null ) {
+ response.setError( "Application not found" );
- return new JSONWithPadding( response, callback );
++ return response;
+ }
+
+ if ( targetUserId == null ) {
+ response.setError( "User not found" );
- return new JSONWithPadding( response, callback );
++ return response;
+ }
+
+ final CredentialsInfo credentialsInfo = management.getAppUserCredentialsInfo( applicationId, targetUserId );
+
+
+ response.setProperty( "credentials", credentialsInfo );
+
+
- return new JSONWithPadding( response, callback );
++ return response;
+ }
+
+
+
+ @PUT
+ @RequireSystemAccess
+ @Path("credentials")
- public JSONWithPadding setUserCredentials( @Context UriInfo ui, Map<String, Object> json,
++ public ApiResponse setUserCredentials( @Context UriInfo ui, Map<String, Object> json,
+ @QueryParam("callback") @DefaultValue("callback") String callback )
+ throws Exception {
+
+ logger.info( "UserResource.setUserCredentials" );
+
+ if ( json == null ) {
+ return null;
+ }
+
+ ApiResponse response = createApiResponse();
+ response.setAction( "set user credentials" );
+ Map<String, Object> credentialsJson = ( Map<String, Object> ) json.get( "credentials" );
+
+
+ if ( credentialsJson == null ) {
+ throw new IllegalArgumentException( "credentials sub object is required" );
+ }
+
+ final CredentialsInfo credentials = CredentialsInfo.fromJson( credentialsJson );
+
+ UUID applicationId = getApplicationId();
+ UUID targetUserId = getUserUuid();
+
+ if ( targetUserId == null ) {
+ response.setError( "User not found" );
- return new JSONWithPadding( response, callback );
++ return response;
+ }
+
+
+ management.setAppUserCredentialsInfo( applicationId, targetUserId, credentials );
+
+
- return new JSONWithPadding( response, callback );
++ return response;
+ }
+
@POST
@Path("password")
@@@ -202,9 -292,7 +302,9 @@@
@POST
@Path("sendpin")
- public JSONWithPadding postSendPin( @Context UriInfo ui,
+ @JSONP
- @Produces({MediaType.APPLICATION_JSON, "application/javascript"})
++ @Produces({ MediaType.APPLICATION_JSON, "application/javascript"})
+ public ApiResponse postSendPin( @Context UriInfo ui,
@QueryParam("callback") @DefaultValue("callback") String callback )
throws Exception {
return sendPin( ui, callback );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/b422364e/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationsResource.java
----------------------------------------------------------------------
diff --cc stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationsResource.java
index 28d8c87,ac07aaa..f48168e
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationsResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationsResource.java
@@@ -17,34 -17,29 +17,40 @@@
package org.apache.usergrid.rest.management.organizations;
-import java.util.*;
-import javax.ws.rs.*;
-import javax.ws.rs.core.Context;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.UriInfo;
+import com.fasterxml.jackson.jaxrs.json.annotation.JSONP;
+import com.google.common.base.Preconditions;
+import org.apache.commons.lang.StringUtils;
+import org.apache.usergrid.management.ApplicationCreator;
+import org.apache.usergrid.management.OrganizationInfo;
+import org.apache.usergrid.management.OrganizationOwnerInfo;
+import org.apache.usergrid.management.exceptions.ManagementException;
+import org.apache.usergrid.persistence.index.query.Identifier;
+import org.apache.usergrid.rest.AbstractContextResource;
+import org.apache.usergrid.rest.ApiResponse;
import org.apache.usergrid.rest.RootResource;
import org.apache.usergrid.rest.management.ManagementResource;
+import org.apache.usergrid.rest.security.annotations.RequireOrganizationAccess;
+ import org.apache.usergrid.rest.security.annotations.RequireSystemAccess;
++
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component;
-import org.apache.usergrid.management.ApplicationCreator;
-import org.apache.usergrid.management.OrganizationInfo;
-import org.apache.usergrid.management.OrganizationOwnerInfo;
-import org.apache.usergrid.management.exceptions.ManagementException;
-import org.apache.usergrid.rest.AbstractContextResource;
-import org.apache.usergrid.rest.ApiResponse;
-import org.apache.usergrid.rest.security.annotations.RequireOrganizationAccess;
-import org.apache.commons.lang.StringUtils;
-import com.google.common.base.Preconditions;
-import com.sun.jersey.api.json.JSONWithPadding;
+
+import javax.ws.rs.*;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.UriInfo;
++
++import java.util.ArrayList;
++import java.util.HashMap;
++import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+
+import static org.apache.usergrid.rest.exceptions.SecurityException.mappableSecurityException;
+import static org.apache.usergrid.security.shiro.utils.SubjectUtils.isPermittedAccessToOrganization;
@Component( "org.apache.usergrid.rest.management.organizations.OrganizationsResource" )
@@@ -67,7 -62,34 +73,34 @@@ public class OrganizationsResource exte
public OrganizationsResource() {
}
+
+ @GET
+ @RequireSystemAccess
- public JSONWithPadding getAllOrganizations() throws Exception{
++ public ApiResponse getAllOrganizations() throws Exception{
+
+ ApiResponse response = createApiResponse();
+ List<OrganizationInfo> orgs = management.getOrganizations(null, 10000);
+ List<Object> jsonOrgList = new ArrayList<>();
+
+ for(OrganizationInfo org: orgs){
+
+
+ Map<String, Object> jsonOrg = new HashMap<>();
+ Map<String, UUID> apps = management.getApplicationsForOrganization(org.getUuid()).inverse();
+
+ jsonOrg.put("name", org.getName());
+ jsonOrg.put("uuid", org.getUuid());
+ jsonOrg.put("properties", org.getProperties());
+ jsonOrg.put("applications", apps);
+ jsonOrgList.add(jsonOrg);
+ }
+
+ response.setProperty("organizations", jsonOrgList);
- return new JSONWithPadding(response);
++
++ return response;
+ }
+
@Path(RootResource.ORGANIZATION_ID_PATH)
- @RequireOrganizationAccess
public OrganizationResource getOrganizationById( @Context UriInfo ui,
@PathParam( "organizationId" ) String organizationIdStr )
throws Exception {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/b422364e/stack/rest/src/main/java/org/apache/usergrid/rest/system/ConnectionResource.java
----------------------------------------------------------------------
diff --cc stack/rest/src/main/java/org/apache/usergrid/rest/system/ConnectionResource.java
index 0000000,14b79f3..ff2a739
mode 000000,100644..100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/system/ConnectionResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/system/ConnectionResource.java
@@@ -1,0 -1,218 +1,216 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ package org.apache.usergrid.rest.system;
+
+
+ import java.util.HashMap;
+ import java.util.Map;
+ import java.util.UUID;
+ import java.util.concurrent.TimeUnit;
+ import java.util.concurrent.atomic.AtomicLong;
+
+ import javax.ws.rs.DefaultValue;
+ import javax.ws.rs.GET;
+ import javax.ws.rs.POST;
+ import javax.ws.rs.Path;
+ import javax.ws.rs.PathParam;
+ import javax.ws.rs.Produces;
+ import javax.ws.rs.QueryParam;
+ import javax.ws.rs.core.MediaType;
+
+ import org.slf4j.Logger;
+ import org.slf4j.LoggerFactory;
+ import org.springframework.context.annotation.Scope;
+ import org.springframework.stereotype.Component;
+
+ import org.apache.usergrid.corepersistence.service.ConnectionService;
+ import org.apache.usergrid.corepersistence.service.ConnectionServiceImpl;
+ import org.apache.usergrid.corepersistence.service.StatusService;
+ import org.apache.usergrid.corepersistence.util.CpNamingUtils;
+ import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+ import org.apache.usergrid.persistence.index.query.Identifier;
+ import org.apache.usergrid.persistence.index.utils.UUIDUtils;
+ import org.apache.usergrid.persistence.model.util.UUIDGenerator;
+ import org.apache.usergrid.rest.AbstractContextResource;
+ import org.apache.usergrid.rest.ApiResponse;
+ import org.apache.usergrid.rest.RootResource;
+ import org.apache.usergrid.rest.security.annotations.RequireSystemAccess;
+
+ import com.google.common.base.Preconditions;
-import com.sun.jersey.api.json.JSONWithPadding;
+
+ import rx.Observable;
-import rx.functions.Action1;
+ import rx.schedulers.Schedulers;
+
+
+ /**
+ * system/index/otherstuff
+ */
+ @Component
+ @Scope( "singleton" )
+ @Produces( {
+ MediaType.APPLICATION_JSON, "application/javascript", "application/x-javascript", "text/ecmascript",
+ "application/ecmascript", "text/jscript"
+ } )
+ public class ConnectionResource extends AbstractContextResource {
+
+ private static final Logger logger = LoggerFactory.getLogger( ConnectionResource.class );
+
+ public ConnectionResource() {
+ super();
+ }
+
+
+ @RequireSystemAccess
+ @POST
+ @Path( "dedup/" + RootResource.APPLICATION_ID_PATH )
- public JSONWithPadding rebuildIndexesPost( @PathParam( "applicationId" ) String applicationIdStr,
- @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
++ public ApiResponse rebuildIndexesPost( @PathParam( "applicationId" ) String applicationIdStr,
++ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+ throws Exception {
+
+
+ logger.info( "Rebuilding all applications" );
+
+ final UUID applicationId = UUIDUtils.tryGetUUID( applicationIdStr );
+
+ Preconditions.checkNotNull( applicationId, "applicationId must be specified" );
+
+ return executeAndCreateResponse( applicationId, callback );
+ }
+
+
+ @RequireSystemAccess
+ @GET
+ @Path( "dedup/{jobId: " + Identifier.UUID_REX + "}" )
- public JSONWithPadding rebuildIndexesGet( @PathParam( "jobId" ) String jobId,
- @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
++ public ApiResponse rebuildIndexesGet( @PathParam( "jobId" ) String jobId,
++ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+ throws Exception {
+ logger.info( "Getting status for index jobs" );
+
+ Preconditions.checkNotNull( jobId, "query param jobId must not be null" );
+
+
+ final UUID jobUUID = UUIDUtils.tryGetUUID( jobId );
+
+ final StatusService.JobStatus
+ job = getStatusService().getStatus( CpNamingUtils.MANAGEMENT_APPLICATION_ID, jobUUID ).toBlocking().lastOrDefault(
+ null );
+
+ Preconditions.checkNotNull( job, "job with id '" + jobId + "' does not exist" );
+
+
+ return createResult( job, callback );
+ }
+
+
+ private ConnectionService getConnectionService() {
+ return injector.getInstance( ConnectionServiceImpl.class );
+ }
+
+
+ private StatusService getStatusService() {
+ return injector.getInstance( StatusService.class );
+ }
+
+
+
+ /**
+ * Execute the request and return the response.
+ */
- private JSONWithPadding executeAndCreateResponse( final UUID applicationId, final String callback ) {
++ private ApiResponse executeAndCreateResponse( final UUID applicationId, final String callback ) {
+
+ final Observable<ApplicationScope> applicationScopeObservable =
+ Observable.just( CpNamingUtils.getApplicationScope( applicationId ) );
+
+ final UUID jobId = UUIDGenerator.newTimeUUID();
+
+ final StatusService statusService = getStatusService();
+ final ConnectionService connectionService = getConnectionService();
+
+ final AtomicLong count = new AtomicLong( 0 );
+
+ //start de duping and run in the background
+ connectionService.deDupeConnections( applicationScopeObservable ).buffer( 10, TimeUnit.SECONDS, 1000 )
+ .doOnNext(buffer -> {
+
+
+ final long runningTotal = count.addAndGet(buffer.size());
+
+ final Map<String, Object> status = new HashMap<String, Object>() {{
+ put("countProcessed", runningTotal);
+ put("updatedTimestamp", System.currentTimeMillis());
+ }};
+
+ statusService.setStatus(CpNamingUtils.MANAGEMENT_APPLICATION_ID, jobId,
+ StatusService.Status.INPROGRESS, status).toBlocking().lastOrDefault(null);
+ }).doOnSubscribe(() -> {
+
+ statusService.setStatus(CpNamingUtils.MANAGEMENT_APPLICATION_ID,
+ jobId, StatusService.Status.STARTED, new HashMap<>()).toBlocking().lastOrDefault(null);
+
+ }).doOnCompleted(() -> {
+
+ final long runningTotal = count.get();
+
+ final Map<String, Object> status = new HashMap<String, Object>() {{
+ put("countProcessed", runningTotal);
+ put("updatedTimestamp", System.currentTimeMillis());
+ }};
+
+ statusService.setStatus(CpNamingUtils.MANAGEMENT_APPLICATION_ID,
+ jobId, StatusService.Status.COMPLETE, status).toBlocking().lastOrDefault(null);
+
+ }).doOnError( (throwable) -> {
+ logger.error("Error deduping connections", throwable);
+
+ final Map<String, Object> status = new HashMap<String, Object>() {{
+ put("error", throwable.getMessage() );
+ }};
+
+ statusService.setStatus(CpNamingUtils.MANAGEMENT_APPLICATION_ID,
+ jobId, StatusService.Status.FAILED, status).toBlocking().lastOrDefault(null);;
+
+ } ).subscribeOn(Schedulers.newThread()).subscribe();
+
+
+ final StatusService.JobStatus status =
+ new StatusService.JobStatus( jobId, StatusService.Status.STARTED, new HashMap<>( ) );
+
+ return createResult( status, callback );
+ }
+
+
+ /**
+ * Create a response with the specified data.
+ * @param jobStatus
+ * @param callback
+ * @return
+ */
- private JSONWithPadding createResult(final StatusService.JobStatus jobStatus, final String callback){
++ private ApiResponse createResult(final StatusService.JobStatus jobStatus, final String callback){
+
+ final ApiResponse response = createApiResponse();
+
+ response.setAction( "de-dup connections" );
+ response.setProperty( "status", jobStatus );
+ response.setSuccess();
+
- return new JSONWithPadding( response, callback );
++ return response;
+ }
+ }
+
+
+
http://git-wip-us.apache.org/repos/asf/usergrid/blob/b422364e/stack/rest/src/main/java/org/apache/usergrid/rest/system/DatabaseResource.java
----------------------------------------------------------------------
diff --cc stack/rest/src/main/java/org/apache/usergrid/rest/system/DatabaseResource.java
index b261c96,42a63ca..f3ce8b7
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/system/DatabaseResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/system/DatabaseResource.java
@@@ -34,8 -31,12 +34,10 @@@ import org.slf4j.LoggerFactory
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component;
+ import org.apache.usergrid.rest.AbstractContextResource;
+ import org.apache.usergrid.rest.ApiResponse;
import org.apache.usergrid.rest.security.annotations.RequireSystemAccess;
-import com.sun.jersey.api.json.JSONWithPadding;
-
@Component
@Scope( "singleton" )
http://git-wip-us.apache.org/repos/asf/usergrid/blob/b422364e/stack/rest/src/main/java/org/apache/usergrid/rest/system/IndexResource.java
----------------------------------------------------------------------
diff --cc stack/rest/src/main/java/org/apache/usergrid/rest/system/IndexResource.java
index 0000000,8e2946c..bbbe8b3
mode 000000,100644..100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/system/IndexResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/system/IndexResource.java
@@@ -1,0 -1,331 +1,353 @@@
+ /*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. The ASF licenses this file to You
+ * * under the Apache License, Version 2.0 (the "License"); you may not
+ * * use this file except in compliance with the License.
+ * * You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License. For additional information regarding
+ * * copyright in this work, please see the NOTICE file in the top level
+ * * directory of this distribution.
+ *
+ */
+
+ package org.apache.usergrid.rest.system;
+
+
++import com.fasterxml.jackson.jaxrs.json.annotation.JSONP;
+ import com.google.common.base.Preconditions;
-import com.sun.jersey.api.json.JSONWithPadding;
+ import org.apache.usergrid.corepersistence.index.ReIndexRequestBuilder;
+ import org.apache.usergrid.corepersistence.index.ReIndexRequestBuilderImpl;
+ import org.apache.usergrid.corepersistence.index.ReIndexService;
+ import org.apache.usergrid.persistence.EntityManager;
+ import org.apache.usergrid.persistence.index.utils.ConversionUtils;
+ import org.apache.usergrid.persistence.index.utils.UUIDUtils;
+ import org.apache.usergrid.rest.AbstractContextResource;
+ import org.apache.usergrid.rest.ApiResponse;
+ import org.apache.usergrid.rest.RootResource;
+ import org.apache.usergrid.rest.security.annotations.RequireSystemAccess;
+ import org.slf4j.Logger;
+ import org.slf4j.LoggerFactory;
+ import org.springframework.context.annotation.Scope;
+ import org.springframework.stereotype.Component;
+
+ import javax.ws.rs.*;
+ import javax.ws.rs.core.Context;
+ import javax.ws.rs.core.MediaType;
+ import javax.ws.rs.core.UriInfo;
+ import java.util.HashMap;
+ import java.util.Map;
+ import java.util.UUID;
+
+
+ /**
+ * system/index/otherstuff
+ */
+ @Component
+ @Scope( "singleton" )
+ @Produces( {
+ MediaType.APPLICATION_JSON, "application/javascript", "application/x-javascript", "text/ecmascript",
+ "application/ecmascript", "text/jscript"
+ } )
+ public class IndexResource extends AbstractContextResource {
+
+ private static final Logger logger = LoggerFactory.getLogger( IndexResource.class );
+ private static final String UPDATED_FIELD = "updated";
+
+
+
+ public IndexResource() {
+ super();
+ }
+
+
+ @RequireSystemAccess
+ @POST
+ @Path( "rebuild" )
- public JSONWithPadding rebuildIndexesPost( @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
++ public ApiResponse rebuildIndexesPost( @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+ throws Exception {
+
+
+ logger.info("Rebuilding all applications");
+
+ final ReIndexRequestBuilder request = createRequest();
+
+ return executeAndCreateResponse( request, callback );
+ }
+
+ @RequireSystemAccess
+ @GET
+ @Path( "rebuild/{jobId}" )
- public JSONWithPadding rebuildIndexesGet(@PathParam( "jobId" ) String jobId, @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
++ @JSONP
++ @Produces({ MediaType.APPLICATION_JSON, "application/javascript" })
++ public ApiResponse rebuildIndexesGet(
++ @PathParam( "jobId" ) String jobId,
++ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
++
++
+ throws Exception {
+ logger.info("Getting status for index jobs");
+
+ Preconditions
+ .checkNotNull(jobId, "query param jobId must not be null" );
+
+
+ ReIndexService.ReIndexStatus status = getReIndexService().getStatus(jobId);
+
+ final ApiResponse response = createApiResponse();
+
+ response.setAction( "rebuild indexes" );
+ response.setProperty( "jobId", status.getJobId() );
+ response.setProperty( "status", status.getStatus() );
+ response.setProperty( "lastUpdatedEpoch", status.getLastUpdated() );
+ response.setProperty( "numberQueued", status.getNumberProcessed() );
+ response.setSuccess();
+
- return new JSONWithPadding( response, callback );
++ return response;
+ }
+
+ @RequireSystemAccess
+ @PUT
+ @Path( "rebuild" )
- public JSONWithPadding rebuildIndexesPut( final Map<String, Object> payload,
++ @JSONP
++ @Produces({MediaType.APPLICATION_JSON, "application/javascript"})
++ public ApiResponse rebuildIndexesPut( final Map<String, Object> payload,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+ throws Exception {
+
+
+ logger.info( "Resuming rebuilding all applications" );
+ final ReIndexRequestBuilder request = createRequest();
+
+ return executeResumeAndCreateResponse( payload, request, callback );
+ }
+
+
+ @RequireSystemAccess
+ @POST
+ @Path( "rebuild/" + RootResource.APPLICATION_ID_PATH )
- public JSONWithPadding rebuildIndexesPut( @PathParam( "applicationId" ) String applicationIdStr,
++ @JSONP
++ @Produces({MediaType.APPLICATION_JSON, "application/javascript"})
++ public ApiResponse rebuildIndexesPut( @PathParam( "applicationId" ) String applicationIdStr,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback,
+ @QueryParam( "delay" ) @DefaultValue( "10" ) final long delay )
+
+ throws Exception {
+
+
+ logger.info( "Rebuilding application {}", applicationIdStr );
+
+
+ final UUID appId = UUIDUtils.tryExtractUUID( applicationIdStr );
+
+ final ReIndexRequestBuilder request = createRequest().withApplicationId( appId );
+
+ return executeAndCreateResponse( request, callback );
+ }
+
+
+ @RequireSystemAccess
+ @PUT
+ @Path( "rebuild/" + RootResource.APPLICATION_ID_PATH )
- public JSONWithPadding rebuildIndexesPut( final Map<String, Object> payload,
++ @JSONP
++ @Produces({MediaType.APPLICATION_JSON, "application/javascript"})
++ public ApiResponse rebuildIndexesPut( final Map<String, Object> payload,
+ @PathParam( "applicationId" ) String applicationIdStr,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback,
+ @QueryParam( "delay" ) @DefaultValue( "10" ) final long delay )
+
+ throws Exception {
+
+ logger.info( "Resuming rebuilding application {}", applicationIdStr );
+
+ final UUID appId = UUIDUtils.tryExtractUUID( applicationIdStr );
+
+ final ReIndexRequestBuilder request = createRequest().withApplicationId( appId );
+
+ return executeResumeAndCreateResponse( payload, request, callback );
+ }
+
+
+ @RequireSystemAccess
+ @POST
+ @Path( "rebuild/" + RootResource.APPLICATION_ID_PATH + "/{collectionName}" )
- public JSONWithPadding rebuildIndexesPost( @PathParam( "applicationId" ) final String applicationIdStr,
++ @JSONP
++ @Produces({MediaType.APPLICATION_JSON, "application/javascript"})
++ public ApiResponse rebuildIndexesPost( @PathParam( "applicationId" ) final String applicationIdStr,
+ @PathParam( "collectionName" ) final String collectionName,
+ @QueryParam( "reverse" ) @DefaultValue( "false" ) final Boolean reverse,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+ throws Exception {
+
+
+ logger.info( "Rebuilding collection {} in application {}", collectionName, applicationIdStr );
+
+ final UUID appId = UUIDUtils.tryExtractUUID( applicationIdStr );
+
+ final ReIndexRequestBuilder request =
+ createRequest().withApplicationId( appId ).withCollection( collectionName );
+
+ return executeAndCreateResponse( request, callback );
+ }
+
+
+ @RequireSystemAccess
+ @PUT
+ @Path( "rebuild/" + RootResource.APPLICATION_ID_PATH + "/{collectionName}" )
- public JSONWithPadding rebuildIndexesPut( final Map<String, Object> payload,
++ @JSONP
++ @Produces({MediaType.APPLICATION_JSON, "application/javascript"})
++ public ApiResponse rebuildIndexesPut( final Map<String, Object> payload,
+ @PathParam( "applicationId" ) final String applicationIdStr,
+ @PathParam( "collectionName" ) final String collectionName,
+ @QueryParam( "reverse" ) @DefaultValue( "false" ) final Boolean reverse,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+ throws Exception {
+
+ logger.info( "Resuming rebuilding collection {} in application {}", collectionName, applicationIdStr );
+
+ final UUID appId = UUIDUtils.tryExtractUUID( applicationIdStr );
+
+ final ReIndexRequestBuilder request =
+ createRequest().withApplicationId( appId ).withCollection( collectionName );
+
+ return executeResumeAndCreateResponse( payload, request, callback );
+ }
+
+
+ @RequireSystemAccess
+ @POST
+ @Path( "rebuild/management" )
- public JSONWithPadding rebuildInternalIndexesPost(
++ @JSONP
++ @Produces({MediaType.APPLICATION_JSON, "application/javascript"})
++ public ApiResponse rebuildInternalIndexesPost(
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback ) throws Exception {
+
+
+ final UUID managementAppId = emf.getManagementAppId();
+
+ logger.info( "Rebuilding management application with id {} ", managementAppId );
+ final ReIndexRequestBuilder request = createRequest().withApplicationId( managementAppId );
+
+ return executeAndCreateResponse( request, callback );
+ }
+
+
+ @RequireSystemAccess
+ @PUT
+ @Path( "rebuild/management" )
- public JSONWithPadding rebuildInternalIndexesPut( final Map<String, Object> payload,
++ @JSONP
++ @Produces({MediaType.APPLICATION_JSON, "application/javascript"})
++ public ApiResponse rebuildInternalIndexesPut( final Map<String, Object> payload,
+ @QueryParam( "callback" ) @DefaultValue( "callback" )
+ String callback ) throws Exception {
+
+
+ final UUID managementAppId = emf.getManagementAppId();
+
+ logger.info( "Resuming rebuilding management application with id {} ", managementAppId );
+ final ReIndexRequestBuilder request = createRequest().withApplicationId( managementAppId );
+
+ return executeResumeAndCreateResponse( payload, request, callback );
+ }
+
+
+ @RequireSystemAccess
+ @POST
+ @Path(RootResource.APPLICATION_ID_PATH)
- public JSONWithPadding addIndex( @Context UriInfo ui,
++ @JSONP
++ @Produces({MediaType.APPLICATION_JSON, "application/javascript"})
++ public ApiResponse addIndex( @Context UriInfo ui,
+ @PathParam( "applicationId" ) final String applicationIdStr,
+ Map<String, Object> config,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+ throws Exception {
+
+ Preconditions
+ .checkNotNull( config, "Payload for config is null, please pass {replicas:int, shards:int} in body" );
+
+ ApiResponse response = createApiResponse();
+
+ if ( !config.containsKey( "replicas" ) || !config.containsKey( "shards" ) ||
+ !( config.get( "replicas" ) instanceof Integer ) || !( config.get( "shards" ) instanceof Integer ) ) {
+ throw new IllegalArgumentException( "body must contains 'replicas' of type int and 'shards' of type int" );
+ }
+
+ if ( !config.containsKey( "indexSuffix" ) ) {
+ throw new IllegalArgumentException( "Please add an indexSuffix to your post" );
+ }
+ final UUID appId = UUIDUtils.tryExtractUUID( applicationIdStr );
+
+ if(appId == null){
+ throw new IllegalArgumentException("app id was not parsed");
+ }
+
+ EntityManager em = emf.getEntityManager(appId);
+ em.addIndex(config.get("indexSuffix").toString(), (int) config.get("shards"),
+ (int) config.get("replicas"), (String) config.get("writeConsistency"));
+ response.setAction( "Add index to alias" );
+
- return new JSONWithPadding( response, callback );
++ return response;
+ }
+
+
+ private ReIndexService getReIndexService() {
+ return injector.getInstance( ReIndexService.class );
+ }
+
+
+ private ReIndexRequestBuilder createRequest() {
+ //TODO: wire this up through spring, and in the future guice.
+ return new ReIndexRequestBuilderImpl();
+ }
+
+
- private JSONWithPadding executeResumeAndCreateResponse( final Map<String, Object> payload,
++ private ApiResponse executeResumeAndCreateResponse( final Map<String, Object> payload,
+ final ReIndexRequestBuilder request,
+ final String callback ) {
+
+ Map<String,Object> newPayload = payload;
+ if(newPayload == null || !payload.containsKey( UPDATED_FIELD )){
+ newPayload = new HashMap<>(1);
+ newPayload.put(UPDATED_FIELD,0);
+ }
+
+ Preconditions.checkArgument(newPayload.get(UPDATED_FIELD) instanceof Number,
+ "You must specified the field \"updated\" in the payload and it must be a timestamp" );
+
+ //add our updated timestamp to the request
+ if ( newPayload.containsKey( UPDATED_FIELD ) ) {
+ final long timestamp = ConversionUtils.getLong(newPayload.get(UPDATED_FIELD));
+ request.withStartTimestamp( timestamp );
+ }
+
+ return executeAndCreateResponse( request, callback );
+ }
+
+
+ /**
+ * Execute the request and return the response.
+ */
- private JSONWithPadding executeAndCreateResponse( final ReIndexRequestBuilder request, final String callback ) {
++ private ApiResponse executeAndCreateResponse( final ReIndexRequestBuilder request, final String callback ) {
+
+
+ final ReIndexService.ReIndexStatus status = getReIndexService().rebuildIndex( request );
+
+ final ApiResponse response = createApiResponse();
+
+ response.setAction( "rebuild indexes" );
+ response.setProperty( "jobId", status.getJobId() );
+ response.setProperty( "status", status.getStatus() );
+ response.setProperty( "lastUpdatedEpoch", status.getLastUpdated() );
+ response.setProperty( "numberQueued", status.getNumberProcessed() );
+ response.setSuccess();
+
- return new JSONWithPadding( response, callback );
++ return response;
+ }
+ }
http://git-wip-us.apache.org/repos/asf/usergrid/blob/b422364e/stack/rest/src/main/java/org/apache/usergrid/rest/system/MigrateResource.java
----------------------------------------------------------------------
diff --cc stack/rest/src/main/java/org/apache/usergrid/rest/system/MigrateResource.java
index 0000000,c5a6dbc..7a6100b
mode 000000,100644..100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/system/MigrateResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/system/MigrateResource.java
@@@ -1,0 -1,270 +1,277 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ package org.apache.usergrid.rest.system;
+
+
-import java.util.Map;
-import java.util.Set;
-
-import javax.ws.rs.*;
-import javax.ws.rs.core.Context;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.UriInfo;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.context.annotation.Scope;
-import org.springframework.stereotype.Component;
-
++import com.fasterxml.jackson.databind.node.JsonNodeFactory;
++import com.fasterxml.jackson.databind.node.ObjectNode;
++import com.fasterxml.jackson.jaxrs.json.annotation.JSONP;
++import com.google.common.base.Preconditions;
++import com.google.inject.Injector;
+ import org.apache.usergrid.persistence.core.migration.data.DataMigrationManager;
+ import org.apache.usergrid.persistence.core.migration.schema.MigrationManager;
+ import org.apache.usergrid.rest.AbstractContextResource;
+ import org.apache.usergrid.rest.ApiResponse;
+ import org.apache.usergrid.rest.security.annotations.RequireSystemAccess;
++import org.slf4j.Logger;
++import org.slf4j.LoggerFactory;
++import org.springframework.beans.factory.annotation.Autowired;
++import org.springframework.context.annotation.Scope;
++import org.springframework.stereotype.Component;
+
-import com.fasterxml.jackson.databind.node.JsonNodeFactory;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import com.google.common.base.Preconditions;
-import com.google.inject.Injector;
-import com.sun.jersey.api.json.JSONWithPadding;
++import javax.ws.rs.*;
++import javax.ws.rs.core.Context;
++import javax.ws.rs.core.MediaType;
++import javax.ws.rs.core.UriInfo;
++import java.util.Map;
++import java.util.Set;
+
+
+ @Component
+ @Scope( "singleton" )
+ @Produces( {
+ MediaType.APPLICATION_JSON,
+ "application/javascript",
+ "application/x-javascript",
+ "text/ecmascript",
+ "application/ecmascript",
+ "text/jscript"
+ } )
+ public class MigrateResource extends AbstractContextResource {
+
+ private static final Logger logger = LoggerFactory.getLogger( MigrateResource.class );
+
+ public MigrateResource() {
+ logger.info( "SystemResource initialized" );
+ }
+
+ @Autowired
+ private Injector guiceInjector;
+
+ @RequireSystemAccess
+ @PUT
+ @Path( "run" )
- public JSONWithPadding migrateData( @Context UriInfo ui,
++ @JSONP
++ @Produces({MediaType.APPLICATION_JSON, "application/javascript"})
++ public ApiResponse migrateData( @Context UriInfo ui,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+ throws Exception {
+
+ ApiResponse response = createApiResponse();
+ response.setAction( "Migrate Data" );
+ //TODO make this use the task scheduler
+
+
+ final Thread migrate = new Thread() {
+
+ @Override
+ public void run() {
+
+ logger.info( "Migrating Schema" );
+
+ try {
+ getMigrationManager().migrate();
+ }
+ catch ( Exception e ) {
+ logger.error( "Unable to migrate data", e );
+ }
+
+ logger.info( "Migrating Data" );
+
+ try {
+ getDataMigrationManager().migrate();
+ }
+ catch ( Exception e ) {
+ logger.error( "Unable to migrate data", e );
+ }
+ }
+ };
+
+ migrate.setName( "Index migrate data formats" );
+ migrate.setDaemon( true );
+ migrate.start();
+
+ response.setSuccess();
+
- return new JSONWithPadding( response, callback );
++ return response;
+ }
+
+ @RequireSystemAccess
+ @PUT
+ @Path( "run/{pluginName}" )
- public JSONWithPadding migrateData(@PathParam("pluginName") String pluginName , @Context UriInfo ui,
++ @JSONP
++ @Produces({MediaType.APPLICATION_JSON, "application/javascript"})
++ public ApiResponse migrateData(@PathParam("pluginName") String pluginName , @Context UriInfo ui,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+ throws Exception {
+
+ if(!getDataMigrationManager().pluginExists(pluginName)){
+ throw new IllegalArgumentException("Plugin doesn't exits name:"+pluginName);
+ }
+
+ ApiResponse response = createApiResponse();
+ response.setAction( "Migrate Data: "+ pluginName );
+ //TODO make this use the task scheduler
+
+
+
+
+ final Thread migrate = new Thread() {
+
+ @Override
+ public void run() {
+
+ logger.info( "Migrating Data for plugin: " + pluginName );
+
+ try {
+ getDataMigrationManager().migrate(pluginName);
+ }
+ catch ( Exception e ) {
+ logger.error( "Unable to migrate data for plugin: " + pluginName, e );
+ }
+ }
+ };
+
+ migrate.setName( "Index migrate data formats: "+pluginName );
+ migrate.setDaemon( true );
+ migrate.start();
+
+ response.setSuccess();
+
- return new JSONWithPadding( response, callback );
++ return response;
+ }
+
+ @RequireSystemAccess
+ @PUT
+ @Path( "set" )
- public JSONWithPadding setMigrationVersion(
++ @JSONP
++ @Produces({MediaType.APPLICATION_JSON, "application/javascript"})
++ public ApiResponse setMigrationVersion(
+ @Context UriInfo ui, Map<String, Object> json,
+ @QueryParam( "callback" ) @DefaultValue( "" ) String callback )
+ throws Exception {
+
+ logger.debug( "setMigrationVersion" );
+
+ Preconditions.checkNotNull( json, "You must provide a json body" );
+ Preconditions.checkArgument( json.keySet().size() > 0, "You must specify at least one module and version" );
+
+ ApiResponse response = createApiResponse();
+ response.setAction("Set Migration Versions");
+
+ ObjectNode node = JsonNodeFactory.instance.objectNode();
+
+ final DataMigrationManager dataMigrationManager = getDataMigrationManager();
+ final Set<String> plugins = dataMigrationManager.getPluginNames();
+
+ /**
+ * Set the migration version for the plugins specified
+ */
+ for ( final String key : json.keySet() ) {
+
+ int version = ( int ) json.get( key );
+
+ dataMigrationManager.resetToVersion(key, version);
+ }
+
+
+ /**
+ * Echo back a response of the current versions for all plugins
+ */
+ for(final String pluginName: plugins){
+ node.put(pluginName, dataMigrationManager.getCurrentVersion(pluginName));
+ }
+
+
+ response.setData( node );
+ response.setSuccess();
+
- return new JSONWithPadding( response, callback );
++ return response;
+ }
+
+
+ @RequireSystemAccess
+ @GET
+ @Path( "status" )
- public JSONWithPadding migrateStatus(
++ @JSONP
++ @Produces({MediaType.APPLICATION_JSON, "application/javascript"})
++ public ApiResponse migrateStatus(
+ @Context UriInfo ui,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+ throws Exception {
+
+ ApiResponse response = createApiResponse();
+ response.setAction( "Migrate Schema indexes" );
+
+ ObjectNode node = JsonNodeFactory.instance.objectNode();
+
+
+
+ final DataMigrationManager dataMigrationManager = getDataMigrationManager();
+
+ final Set<String> plugins = dataMigrationManager.getPluginNames();
+
+ for(final String pluginName: plugins){
+ node.put( pluginName, dataMigrationManager.getCurrentVersion( pluginName ) );
+ }
+
+ response.setData( node );
+
+ response.setSuccess();
+
- return new JSONWithPadding( response, callback );
++ return response;
+ }
+
+
+ @RequireSystemAccess
+ @GET
+ @Path( "count" )
- public JSONWithPadding migrateCount(
++ @JSONP
++ @Produces({MediaType.APPLICATION_JSON, "application/javascript"})
++ public ApiResponse migrateCount(
+ @Context UriInfo ui,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+ throws Exception {
+
+ ApiResponse response = createApiResponse();
+ response.setAction( "Current entity count in system" );
+
+ response.setProperty( "count", emf.performEntityCount() );
+
+ response.setSuccess();
+
- return new JSONWithPadding( response, callback );
++ return response;
+ }
+
+
+ /**
+ * Get the schema migration manager
+ */
+ private MigrationManager getMigrationManager() {
+ return guiceInjector.getInstance( MigrationManager.class );
+ }
+
+ /**
+ * Get the Data migration manager
+ */
+ private DataMigrationManager getDataMigrationManager() {
+ return guiceInjector.getInstance( DataMigrationManager.class );
+ }
+ }
+
http://git-wip-us.apache.org/repos/asf/usergrid/blob/b422364e/stack/rest/src/main/java/org/apache/usergrid/rest/system/SystemResource.java
----------------------------------------------------------------------
diff --cc stack/rest/src/main/java/org/apache/usergrid/rest/system/SystemResource.java
index 0000000,a5174e6..a83756d
mode 000000,100644..100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/system/SystemResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/system/SystemResource.java
@@@ -1,0 -1,108 +1,121 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+ package org.apache.usergrid.rest.system;
+
+
+ import javax.ws.rs.DefaultValue;
+ import javax.ws.rs.GET;
+ import javax.ws.rs.Path;
+ import javax.ws.rs.Produces;
+ import javax.ws.rs.QueryParam;
+ import javax.ws.rs.core.Context;
+ import javax.ws.rs.core.MediaType;
+ import javax.ws.rs.core.UriInfo;
+
++import org.apache.usergrid.rest.system.ApplicationsResource;
++import org.apache.usergrid.rest.system.DatabaseResource;
++import org.apache.usergrid.rest.system.QueueSystemResource;
+ import org.slf4j.Logger;
+ import org.slf4j.LoggerFactory;
+ import org.springframework.context.annotation.Scope;
+ import org.springframework.stereotype.Component;
+
+ import org.apache.usergrid.rest.AbstractContextResource;
+ import org.apache.usergrid.rest.ApiResponse;
+ import org.apache.usergrid.rest.security.annotations.RequireSystemAccess;
+
-import com.sun.jersey.api.json.JSONWithPadding;
++import javax.ws.rs.*;
++import javax.ws.rs.core.Context;
++import javax.ws.rs.core.MediaType;
++import javax.ws.rs.core.UriInfo;
+
+
+ @Path( "/system" )
+ @Component
+ @Scope( "singleton" )
+ @Produces( {
+ MediaType.APPLICATION_JSON, "application/javascript", "application/x-javascript", "text/ecmascript",
+ "application/ecmascript", "text/jscript"
+ } )
+ public class SystemResource extends AbstractContextResource {
+
+ private static final Logger logger = LoggerFactory.getLogger( SystemResource.class );
+
+
+ public SystemResource() {
+ logger.info( "SystemResource initialized" );
+ }
+
+
+ @RequireSystemAccess
+ @GET
++
+ @Path( "superuser/setup" )
- public JSONWithPadding getSetupSuperuser( @Context UriInfo ui,
- @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
++ @Produces({MediaType.APPLICATION_JSON, "application/javascript"})
++ public ApiResponse getSetupSuperuser( @Context UriInfo ui,
++ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+ throws Exception {
+
+ ApiResponse response = createApiResponse();
+ response.setAction( "superuser setup" );
+
+ logger.info( "Setting up Superuser" );
+
+ try {
+ management.provisionSuperuser();
+ }
+ catch ( Exception e ) {
+ logger.error( "Unable to complete superuser setup", e );
+ }
+
+ response.setSuccess();
+
- return new JSONWithPadding( response, callback );
++ return response;
+ }
+
+
+
+ @Path( "migrate" )
+ public MigrateResource migrate() {
+ return getSubResource( MigrateResource.class );
+ }
+
+
+ @Path( "index" )
+ public IndexResource index() { return getSubResource( IndexResource.class ); }
+
+
+ @Path( "database" )
+ public DatabaseResource database() {
+ return getSubResource( DatabaseResource.class );
+ }
+
++ @Path( "queue" )
++ public QueueSystemResource queue() {
++ return getSubResource( QueueSystemResource.class );
++ }
++
+ @Path( "applications" )
+ public ApplicationsResource applications() {
+ return getSubResource( ApplicationsResource.class );
+ }
+
+
+ @Path( "connection" )
+ public ConnectionResource connection() { return getSubResource( ConnectionResource.class ); }
+
+ }
http://git-wip-us.apache.org/repos/asf/usergrid/blob/b422364e/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/UserResourceIT.java
----------------------------------------------------------------------
diff --cc stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/UserResourceIT.java
index 0df2416,f258f94..eddf7ed
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/UserResourceIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/UserResourceIT.java
@@@ -18,6 -18,6 +18,10 @@@ package org.apache.usergrid.rest.applic
import com.fasterxml.jackson.databind.JsonNode;
++import com.sun.jersey.api.client.UniformInterfaceException;
++import com.sun.jersey.api.client.WebResource;
++import com.sun.jersey.api.client.filter.HTTPBasicAuthFilter;
++
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@@ -27,6 -27,6 +31,8 @@@ import java.util.UUID
import org.apache.usergrid.rest.test.resource.AbstractRestIT;
import org.apache.usergrid.rest.test.resource.endpoints.CollectionEndpoint;
import org.apache.usergrid.rest.test.resource.model.*;
++
++import org.glassfish.jersey.client.authentication.HttpAuthenticationFeature;
import org.junit.Before;
import org.junit.Test;
import org.slf4j.Logger;
@@@ -36,10 -36,16 +42,14 @@@ import org.slf4j.LoggerFactory
import org.apache.usergrid.rest.applications.utils.UserRepo;
import org.apache.usergrid.utils.UUIDUtils;
-import com.sun.jersey.api.client.ClientResponse.Status;
-import com.sun.jersey.api.client.GenericType;
-import com.sun.jersey.api.client.UniformInterfaceException;
-import com.sun.jersey.api.client.WebResource;
-import com.sun.jersey.api.client.filter.HTTPBasicAuthFilter;
-
++import javax.ws.rs.BadRequestException;
+import javax.ws.rs.ClientErrorException;
++import javax.ws.rs.client.WebTarget;
+import javax.ws.rs.core.Response;
import java.io.IOException;
+ import javax.ws.rs.core.MediaType;
+
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
@@@ -1129,4 -1109,105 +1139,111 @@@ public class UserResourceIT extends Abs
assertEquals(response.getResponse().getEntities().get(0).get("uuid").toString(), userId.toString());
}
+
+
+
+ @Test
+ public void testCredentialsTransfer() throws Exception {
+
+ usersResource.post(new User("test_1", "Test1 User", "test_1@test.com", "test123")); // client.setApiUrl(apiUrl);
+ refreshIndex();
+
+ //Entity appInfo = this.app().get().getResponse().getEntities().get(0);
+
+ Token token = this.app().token().post(new Token("test_1", "test123"));
+
+ assertNotNull(token.getAccessToken());
+
+ final String superUserName = this.clientSetup.getSuperuserName();
+ final String superUserPassword = this.clientSetup.getSuperuserPassword();
+
+
+ //get the credentials info
+ final CollectionEndpoint collection = userResource.entity("test_1").collection( "credentials" );
+
- final WebResource resource = collection.getResource();
++ final WebTarget resource = collection.getTarget();
++
++
++ final HttpAuthenticationFeature httpBasicAuth = HttpAuthenticationFeature.basicBuilder()
++ .credentials( superUserName, superUserPassword ).build();
+
- resource.addFilter( new HTTPBasicAuthFilter(superUserName, superUserPassword) );
+
++ final ApiResponse response = resource.register( httpBasicAuth ).request()
++ .accept( MediaType.APPLICATION_JSON ).get(
++ org.apache.usergrid.rest.test.resource.model.ApiResponse.class );
+
+
- final ApiResponse response = resource.type( MediaType.APPLICATION_JSON_TYPE)
- .accept( MediaType.APPLICATION_JSON ).get( org.apache.usergrid.rest.test.resource.model.ApiResponse.class );
+
+
+ //now get the credentials sub object
+
+ final Map<String, Object> credentials = ( Map<String, Object> ) response.getProperties().get( "credentials" );
+
+
+
+ //get out the hash and change it so we can validate
+ final String originalSecret = ( String ) credentials.get( "secret" );
+
+
+ //here we modify the hash a little, this way we can break password validation, then re-set it to ensure we're actually updating the credentials info correctly.
+ final String borkedSecret = originalSecret.substring( 0, originalSecret.length() -1 );
+
+ credentials.put( "secret", borkedSecret );
+
+ //now PUT it
+
+
+ final Map<String, Map<String, Object>> wrapper = new HashMap<>( );
+ wrapper.put( "credentials", credentials );
+
- final WebResource putResource = collection.getResource();
++ final WebTarget putResource = collection.getTarget();
+
- putResource.addFilter( new HTTPBasicAuthFilter(superUserName, superUserPassword) );
+
+
- putResource.type( MediaType.APPLICATION_JSON_TYPE)
- .accept( MediaType.APPLICATION_JSON ).put(
- org.apache.usergrid.rest.test.resource.model.ApiResponse.class, wrapper );
++ putResource.register( httpBasicAuth ).request()
++ .accept( MediaType.APPLICATION_JSON )
++ .put( javax.ws.rs.client.Entity.json(wrapper), org.apache.usergrid.rest.test.resource.model.ApiResponse.class );
+
+
+ //now try to get a password, it should fail because the hash is no longer correct
+
+ int status = 0;
+
+ // bad access token
+ try {
+ this.app().token().post(new Token("test_1", "test123"));
+ fail("Should have thrown an exception");
- } catch (UniformInterfaceException uie) {
++ } catch (BadRequestException uie) {
+ status = uie.getResponse().getStatus();
- log.info("Error Response Body: " + uie.getResponse().getEntity(String.class));
++ log.info("Error Response Body: {}" , uie.getResponse().getEntity());
+ }
+
- assertEquals(Status.BAD_REQUEST.getStatusCode(), status);
++ assertEquals( Response.Status.BAD_REQUEST.getStatusCode(), status);
+
+
+ //now put the correct one
+
+
+ credentials.put( "secret", originalSecret );
+
+
- final WebResource putResource2 = collection.getResource();
++ final WebTarget putResource2 = collection.getTarget();
++
++
++
++ putResource2.register( httpBasicAuth ).request()
++ .accept( MediaType.APPLICATION_JSON )
++ .put( javax.ws.rs.client.Entity.json( wrapper ),
++ org.apache.usergrid.rest.test.resource.model.ApiResponse.class );
+
- putResource2.addFilter( new HTTPBasicAuthFilter( superUserName, superUserPassword ) );
+
+
- putResource2.type( MediaType.APPLICATION_JSON_TYPE)
- .accept( MediaType.APPLICATION_JSON ).put(
- org.apache.usergrid.rest.test.resource.model.ApiResponse.class, wrapper );
+
+
+ //now auth, should be good
+ final Token nextToken = this.app().token().post(new Token("test_1", "test123"));
+
+ assertNotNull( nextToken.getAccessToken() );
+
+ }
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/b422364e/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
----------------------------------------------------------------------
[36/50] [abbrv] usergrid git commit: Add call to database bootstrap
Posted by sn...@apache.org.
Add call to database bootstrap
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/9b589d37
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/9b589d37
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/9b589d37
Branch: refs/heads/asf-site
Commit: 9b589d37bba835cdc8fc8912919f7929fa1bfb21
Parents: 8737f69
Author: Dave Johnson <sn...@apache.org>
Authored: Fri Oct 30 13:49:25 2015 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Fri Oct 30 13:49:25 2015 -0400
----------------------------------------------------------------------
stack/scripts/multitenant_migrate.py | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/9b589d37/stack/scripts/multitenant_migrate.py
----------------------------------------------------------------------
diff --git a/stack/scripts/multitenant_migrate.py b/stack/scripts/multitenant_migrate.py
index 6955de7..5f1918e 100644
--- a/stack/scripts/multitenant_migrate.py
+++ b/stack/scripts/multitenant_migrate.py
@@ -226,6 +226,7 @@ class Migrate:
# Init the migration system as this is the first migration done on the cluster
self.run_database_setup()
+ self.run_database_bootstrap()
migration_system_updated = self.is_migration_system_updated()
[16/50] [abbrv] usergrid git commit: USERGRID-1044: for
org/app/collection create, default to testConfig settings
Posted by sn...@apache.org.
USERGRID-1044: for org/app/collection create, default to testConfig settings
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/fc23b79e
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/fc23b79e
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/fc23b79e
Branch: refs/heads/asf-site
Commit: fc23b79e9a8a08bc57d25e3e60a9718304f8f1c0
Parents: 0389744
Author: Mike Dunker <md...@apigee.com>
Authored: Tue Oct 27 16:39:33 2015 -0700
Committer: Mike Dunker <md...@apigee.com>
Committed: Tue Oct 27 16:39:33 2015 -0700
----------------------------------------------------------------------
stack/loadtests/runOrgAppSetup.sh | 17 ++++++++++-------
1 file changed, 10 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/fc23b79e/stack/loadtests/runOrgAppSetup.sh
----------------------------------------------------------------------
diff --git a/stack/loadtests/runOrgAppSetup.sh b/stack/loadtests/runOrgAppSetup.sh
index 19936b7..b26eb37 100755
--- a/stack/loadtests/runOrgAppSetup.sh
+++ b/stack/loadtests/runOrgAppSetup.sh
@@ -29,6 +29,9 @@ if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi
#ENTITY_TYPE=
#ENTITY_PREFIX=
#ENTITY_SEED= #may be overridden on command line
+#ORG= #may be overridden on command line
+#APP= #may be overridden on command line
+#COLLECTION= #may be overridden on command line
#RETRY_COUNT=
#ENTITY_PROGRESS_COUNT=
#CONSTANT_USERS_PER_SEC=
@@ -36,14 +39,14 @@ if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi
die() { echo "$@" 1>&2 ; exit 1; }
-[ "$#" -ge 2 ] || die "At least 2 arguments required, $# provided. Example is $0 ORG APP [COLLECTION [SANDBOX_COLLECTION (true/false)]]"
+[ "$#" -ge 1 ] || die "At least 1 argument required, $# provided. Example is $0 SANDBOX_COLLECTION(true/false) [ORG [APP [COLLECTION]]]"
-ORG="$1"
-APP="$2"
-COLLECTION="gatlingitems"
-[ "$#" -ge 3 ] && COLLECTION="$3"
-SANDBOX_COLLECTION=true
-[ "$#" -ge 4 ] && SANDBOX_COLLECTION="$4"
+
+SANDBOX_COLLECTION="$1"
+# org, app, and collection can come from testConfig.sh
+[ "$#" -ge 2 ] && ORG="$2"
+[ "$#" -ge 3 ] && APP="$3"
+[ "$#" -ge 4 ] && COLLECTION="$4"
shift $#
[32/50] [abbrv] usergrid git commit: Merge branch '2.1-release' into
multitenant-migration
Posted by sn...@apache.org.
Merge branch '2.1-release' into multitenant-migration
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/b816a188
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/b816a188
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/b816a188
Branch: refs/heads/asf-site
Commit: b816a188bade82ec1044e4dfb80220da67182b3c
Parents: 8af152e c652171
Author: Dave Johnson <sn...@apache.org>
Authored: Thu Oct 29 20:24:14 2015 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Thu Oct 29 20:24:14 2015 -0400
----------------------------------------------------------------------
.../corepersistence/index/IndexServiceImpl.java | 2 +-
.../persistence/index/CandidateResult.java | 11 +-
.../persistence/index/EntityIndexBatch.java | 2 +-
.../usergrid/persistence/index/IndexFig.java | 6 +
.../index/impl/DeIndexOperation.java | 5 +
.../index/impl/EsEntityIndexBatchImpl.java | 34 ++++-
.../index/impl/EsEntityIndexImpl.java | 149 +++++++++++--------
.../persistence/index/impl/IndexingUtils.java | 2 +-
.../persistence/index/impl/EntityIndexTest.java | 14 +-
9 files changed, 143 insertions(+), 82 deletions(-)
----------------------------------------------------------------------
[42/50] [abbrv] usergrid git commit: USERGRID-1044: update README
Posted by sn...@apache.org.
USERGRID-1044: update README
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/73dcaebd
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/73dcaebd
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/73dcaebd
Branch: refs/heads/asf-site
Commit: 73dcaebd286fad3edffd0a8e523b31b042b061c2
Parents: 247f796
Author: Mike Dunker <md...@apigee.com>
Authored: Mon Nov 2 08:54:18 2015 -0800
Committer: Mike Dunker <md...@apigee.com>
Committed: Mon Nov 2 08:54:18 2015 -0800
----------------------------------------------------------------------
stack/loadtests/README.md | 12 +++++++++++-
1 file changed, 11 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/73dcaebd/stack/loadtests/README.md
----------------------------------------------------------------------
diff --git a/stack/loadtests/README.md b/stack/loadtests/README.md
index ec34b42..9dc24e9 100644
--- a/stack/loadtests/README.md
+++ b/stack/loadtests/README.md
@@ -9,6 +9,9 @@ The test scripts are found in the top level loadtests directory. Look inside the
###testConfig.sh
Contains defaults that are used for all the other test scripts.
+###runAuditDeleteEntities.sh
+For a specified organization and given CSV file, delete all entities, writing those that fail to delete to another CSV file.
+
###runAuditGetAllAppCollectionEntities.sh
For a specified organization, finds all apps, and for each app, finds all collections and writes a CSV file line containing collection name, UUID, entity name, and modified timestamp for each entity in each collection.
@@ -16,7 +19,7 @@ For a specified organization, finds all apps, and for each app, finds all collec
For a specified organization and application, finds all collections and writes a CSV file line containing collection name, UUID, entity name, and modified timestamp for each entity in each collection.
###runAuditVerifyCollectionEntities.sh
-For a specified organization and given CSV file, verify that all entities are retrievable, writing those that fail to another CSV file.
+For a specified organization and given CSV file, verify that all entities are retrievable via query, writing those that fail to another CSV file, optionally deleting verified entities.
###runCollectionQueryTest.sh
For a given collection, retrieve all entities using a cursor and a query.
@@ -115,6 +118,13 @@ Defaults listed are those that are specified by the Usergrid Gatling code, not n
* getViaQuery (**false**) - retrieve entities via query instead of via name or uuid
* queryParams (**""**) - additional query parameters (currently used for get by entity or by name)
* csvFeedPattern (**"random"**) - pattern to use when feeding from a CSV ("random" is random, "circular" goes through CSV sequentially and restarts from beginning when it reaches the end)
+* unlimitedFeed (**false**) - continue loading with no limit on number of entities (forces interleavedWorkerFeed=true); hit CTRL-c to abort
+* flushCsv (**0**) - if > 0, flush CSV file when that number of CSV entries has been received
+* interleavedWorkerFeed (**false**) - for multiple gatling servers, shard the entities via interleaving rather than splitting into separate chunks; for example, for 10 workers over 1M entities, interleaving would cause one worker to have 1, 11, 21, 31, etc. instead of 1-100000
+* newCsvOnFlush (**false**) - when the output CSV file is flushed, create a new CSV file
+* deleteAfterSuccessfulAudit (**false**) - delete a record that is successfully verified via query
+* usergridRegion (**""**) - send specified region in the UsergridRegion header for all requests -- this allows Edge proxy to call different regions for testing
+* saveInvalidResponse (**false**) - on audit, do not retry but immediately save failure to output CSV file
The following settings are currently not used (were used by deprecated tests, but may be valid in the future):
[25/50] [abbrv] usergrid git commit: Adds the ability to get
credentials info as superuser for user migration
Posted by sn...@apache.org.
Adds the ability to get credentials info as superuser for user migration
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/08e97813
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/08e97813
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/08e97813
Branch: refs/heads/asf-site
Commit: 08e978132375751caf90ebcffde7a20d5e84b206
Parents: 5ed8c7c
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Oct 28 15:27:36 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Thu Oct 29 13:55:38 2015 -0600
----------------------------------------------------------------------
.../rest/applications/users/UserResource.java | 45 ++++++++++++++++++--
.../usergrid/management/ManagementService.java | 2 +
.../cassandra/ManagementServiceImpl.java | 19 +++++++++
3 files changed, 62 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/08e97813/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java
index df88cf0..fb10245 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java
@@ -42,6 +42,11 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component;
+
+import org.apache.amber.oauth2.common.exception.OAuthProblemException;
+import org.apache.amber.oauth2.common.message.OAuthResponse;
+import org.apache.commons.lang.StringUtils;
+
import org.apache.usergrid.management.ActivationState;
import org.apache.usergrid.persistence.CredentialsInfo;
import org.apache.usergrid.persistence.EntityManager;
@@ -52,13 +57,10 @@ import org.apache.usergrid.rest.ApiResponse;
import org.apache.usergrid.rest.applications.ServiceResource;
import org.apache.usergrid.rest.exceptions.RedirectionException;
import org.apache.usergrid.rest.security.annotations.RequireApplicationAccess;
+import org.apache.usergrid.rest.security.annotations.RequireSystemAccess;
import org.apache.usergrid.security.oauth.AccessInfo;
import org.apache.usergrid.security.tokens.exceptions.TokenException;
-import org.apache.amber.oauth2.common.exception.OAuthProblemException;
-import org.apache.amber.oauth2.common.message.OAuthResponse;
-import org.apache.commons.lang.StringUtils;
-
import com.sun.jersey.api.json.JSONWithPadding;
import com.sun.jersey.api.view.Viewable;
@@ -165,6 +167,41 @@ public class UserResource extends ServiceResource {
return new JSONWithPadding( response, callback );
}
+ @GET
+ @RequireSystemAccess
+ @Path("password")
+ public JSONWithPadding getUserPassword(@QueryParam("callback") @DefaultValue("callback") String callback )
+ throws Exception {
+
+ logger.info( "UserResource.setUserPassword" );
+
+
+ final ApiResponse response = createApiResponse();
+ response.setAction( "get user password" );
+
+ final UUID applicationId = getApplicationId();
+ final UUID targetUserId = getUserUuid();
+
+ if ( applicationId == null ) {
+ response.setError( "Application not found" );
+ return new JSONWithPadding( response, callback );
+ }
+
+ if ( targetUserId == null ) {
+ response.setError( "User not found" );
+ return new JSONWithPadding( response, callback );
+ }
+
+ final CredentialsInfo credentialsInfo = management.getAppUserPasswordRaw( applicationId, targetUserId );
+
+
+ response.setProperty( "credentials", credentialsInfo );
+
+
+ return new JSONWithPadding( response, callback );
+ }
+
+
@PUT
@Path("credentials")
http://git-wip-us.apache.org/repos/asf/usergrid/blob/08e97813/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java b/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java
index d69de2e..3f02e5a 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java
@@ -288,6 +288,8 @@ public interface ManagementService {
public void setAppUserPassword( UUID applicationId, UUID userId, String oldPassword, String newPassword )
throws Exception;
+ CredentialsInfo getAppUserPasswordRaw( final UUID applicationId, final UUID userId ) throws Exception;
+
/**
* Set the credentials info into the
* @param applicationId
http://git-wip-us.apache.org/repos/asf/usergrid/blob/08e97813/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
index 2e33539..b633727 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
@@ -2825,6 +2825,25 @@ public class ManagementServiceImpl implements ManagementService {
@Override
+ public CredentialsInfo getAppUserPasswordRaw( final UUID applicationId, final UUID userId ) throws Exception {
+
+ final User user = emf.getEntityManager( applicationId ).get( userId, User.class );
+
+ if(user == null){
+ throw new EntityNotFoundException("Could not find user with id " + userId + " in application" + applicationId );
+ }
+
+ final CredentialsInfo ci = readUserPasswordCredentials( applicationId, userId );
+
+ if ( ci == null ) {
+ throw new EntityNotFoundException("Could not find credentials for user with id " + userId + " in application" + applicationId );
+ }
+
+ return ci;
+ }
+
+
+ @Override
public User verifyAppUserPasswordCredentials( UUID applicationId, String name, String password ) throws Exception {
User user = findUserEntity( applicationId, name );
[13/50] [abbrv] usergrid git commit: Merge branch
'delete-event-updates' into 2.1-release
Posted by sn...@apache.org.
Merge branch 'delete-event-updates' into 2.1-release
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/26860545
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/26860545
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/26860545
Branch: refs/heads/asf-site
Commit: 26860545ece19361569370bc63496a01bb48738d
Parents: e018c1e 76476f1
Author: Todd Nine <tn...@apigee.com>
Authored: Tue Oct 27 14:40:23 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Tue Oct 27 14:40:23 2015 -0600
----------------------------------------------------------------------
.../asyncevents/AmazonAsyncEventService.java | 68 ++++++++++--------
.../asyncevents/AsyncEventService.java | 5 ++
.../asyncevents/InMemoryAsyncEventService.java | 5 ++
.../index/IndexProcessorFig.java | 13 +++-
.../read/traverse/AbstractReadGraphFilter.java | 72 +++++++++++++++++---
.../traverse/ReadGraphCollectionFilter.java | 10 ++-
.../traverse/ReadGraphConnectionFilter.java | 10 ++-
.../impl/stage/NodeDeleteListenerImpl.java | 27 +++-----
8 files changed, 150 insertions(+), 60 deletions(-)
----------------------------------------------------------------------
[50/50] [abbrv] usergrid git commit: Merge branch 'master' into
asf-site
Posted by sn...@apache.org.
Merge branch 'master' into asf-site
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/cbe501af
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/cbe501af
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/cbe501af
Branch: refs/heads/asf-site
Commit: cbe501afabe7c3453595947ead1811dfb28ad2ea
Parents: 0fd9939 c3299bc
Author: Dave Johnson <sn...@apache.org>
Authored: Fri Nov 6 15:37:26 2015 -0500
Committer: Dave Johnson <sn...@apache.org>
Committed: Fri Nov 6 15:37:26 2015 -0500
----------------------------------------------------------------------
.../installation/ug2-deploy-to-tomcat.txt | 2 +
.../docs/installation/ug2-deploy-to-tomcat.html | 2 +
docs/installation/ug2-deploy-to-tomcat.md | 2 +
.../corepersistence/CpEntityManagerFactory.java | 7 +-
.../corepersistence/CpRelationManager.java | 16 +-
.../asyncevents/AmazonAsyncEventService.java | 99 +--
.../asyncevents/AsyncEventService.java | 5 +
.../asyncevents/EventBuilder.java | 25 +-
.../asyncevents/EventBuilderImpl.java | 60 +-
.../index/IndexProcessorFig.java | 15 +-
.../corepersistence/index/IndexServiceImpl.java | 5 +-
.../pipeline/read/FilterResult.java | 7 +
.../read/traverse/AbstractReadGraphFilter.java | 136 +++-
.../read/traverse/EdgeCursorSerializer.java | 8 +-
.../read/traverse/EntityLoadVerifyFilter.java | 24 +-
.../traverse/ReadGraphCollectionFilter.java | 11 +-
.../ReadGraphConnectionByTypeFilter.java | 11 +-
.../traverse/ReadGraphConnectionFilter.java | 11 +-
.../results/ObservableQueryExecutor.java | 7 +
.../service/StatusServiceImpl.java | 3 +-
.../usergrid/persistence/CredentialsInfo.java | 46 ++
.../persistence/ObservableIterator.java | 83 ---
.../index/AsyncIndexServiceTest.java | 3 +-
.../corepersistence/index/IndexServiceTest.java | 12 +-
.../usergrid/corepersistence/index/RxTest.java | 4 +-
.../pipeline/cursor/CursorTest.java | 24 +-
.../service/ConnectionServiceImplTest.java | 5 +-
.../persistence/ApplicationServiceIT.java | 4 +-
.../usergrid/persistence/RebuildIndexTest.java | 118 +++-
.../persistence/core/astyanax/CassandraFig.java | 2 +-
.../core/astyanax/MultiRowColumnIterator.java | 46 +-
.../persistence/core/rx/ObservableIterator.java | 2 +-
.../usergrid/persistence/graph/GraphFig.java | 4 +-
.../persistence/graph/GraphManager.java | 14 +-
.../usergrid/persistence/graph/MarkedEdge.java | 15 +-
.../persistence/graph/guice/GraphModule.java | 11 +-
.../graph/impl/GraphManagerImpl.java | 90 ++-
.../graph/impl/SimpleMarkedEdge.java | 62 +-
.../impl/stage/NodeDeleteListenerImpl.java | 27 +-
.../impl/EdgeSerializationImpl.java | 17 +-
.../impl/shard/AsyncTaskExecutor.java | 34 +
.../graph/serialization/impl/shard/Shard.java | 15 +
.../impl/shard/ShardEntryGroup.java | 13 +-
.../impl/shard/ShardGroupCompaction.java | 4 -
.../impl/shard/ShardGroupDeletion.java | 78 +++
.../impl/shard/impl/AsyncTaskExecutorImpl.java | 53 ++
.../shard/impl/NodeShardAllocationImpl.java | 81 ++-
.../shard/impl/ShardGroupColumnIterator.java | 72 +-
.../shard/impl/ShardGroupCompactionImpl.java | 10 +-
.../impl/shard/impl/ShardGroupDeletionImpl.java | 230 +++++++
.../impl/shard/impl/ShardsColumnIterator.java | 10 +
.../persistence/graph/GraphManagerIT.java | 76 +--
.../persistence/graph/GraphManagerLoadTest.java | 10 +-
.../graph/GraphManagerShardConsistencyIT.java | 382 ++++++++---
.../graph/GraphManagerStressTest.java | 16 +-
.../impl/shard/ShardEntryGroupTest.java | 14 +
.../impl/shard/ShardGroupCompactionTest.java | 30 +-
.../shard/impl/ShardGroupDeletionImplTest.java | 341 +++++++++
stack/corepersistence/pom.xml | 2 +-
.../persistence/index/CandidateResult.java | 11 +-
.../persistence/index/EntityIndexBatch.java | 2 +-
.../usergrid/persistence/index/IndexFig.java | 6 +
.../index/impl/DeIndexOperation.java | 5 +
.../index/impl/EsEntityIndexBatchImpl.java | 34 +-
.../index/impl/EsEntityIndexImpl.java | 188 +++--
.../persistence/index/impl/IndexingUtils.java | 2 +-
.../persistence/index/impl/EntityIndexTest.java | 14 +-
.../usergrid/persistence/queue/QueueFig.java | 2 +-
stack/loadtests/README.md | 14 +-
stack/loadtests/runAuditDeleteEntities.sh | 65 ++
.../runAuditGetAllAppCollectionEntities.sh | 6 +-
.../loadtests/runAuditGetCollectionEntities.sh | 6 +-
.../runAuditVerifyCollectionEntities.sh | 2 +-
stack/loadtests/runLoadEntitiesUnlimited.sh | 1 +
stack/loadtests/runOrgAppSetup.sh | 17 +-
.../datagenerators/FeederGenerator.scala | 16 +-
.../usergrid/enums/ConfigProperties.scala | 8 +-
.../apache/usergrid/enums/ScenarioType.scala | 3 +-
.../apache/usergrid/helpers/Extractors.scala | 18 +-
.../usergrid/scenarios/AuditScenarios.scala | 203 ++++--
.../scenarios/EntityCollectionScenarios.scala | 102 ++-
.../org/apache/usergrid/settings/Settings.scala | 42 +-
.../usergrid/simulations/AuditSimulation.scala | 15 +-
.../simulations/ConfigurableSimulation.scala | 2 +-
stack/loadtests/testConfig.sh | 4 +-
.../org/apache/usergrid/rest/IndexResource.java | 350 ----------
.../apache/usergrid/rest/MigrateResource.java | 275 --------
.../apache/usergrid/rest/SystemResource.java | 111 ---
.../rest/applications/ApplicationResource.java | 50 ++
.../rest/applications/users/UserResource.java | 134 +++-
.../organizations/OrganizationsResource.java | 33 +
.../rest/system/ConnectionResource.java | 216 ++++++
.../usergrid/rest/system/DatabaseResource.java | 2 +
.../usergrid/rest/system/IndexResource.java | 353 ++++++++++
.../usergrid/rest/system/MigrateResource.java | 277 ++++++++
.../usergrid/rest/system/SystemResource.java | 121 ++++
.../collection/users/UserResourceIT.java | 117 ++++
stack/scripts/create_test_data.py | 222 ++++++
stack/scripts/migrate_entity_data.py | 18 +-
stack/scripts/multitenant_migrate.py | 683 +++++++++++++++++++
.../management/AppInfoMigrationPlugin.java | 5 +-
.../usergrid/management/ManagementService.java | 14 +-
.../cassandra/ManagementServiceImpl.java | 43 +-
103 files changed, 4737 insertions(+), 1591 deletions(-)
----------------------------------------------------------------------