You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by to...@apache.org on 2015/09/24 22:15:19 UTC
[1/6] usergrid git commit: Changes implementation to update a
connection to have the timestamp it was last POSTed.
Repository: usergrid
Updated Branches:
refs/heads/two-dot-o-dev 680338c5a -> eda816bb0
Changes implementation to update a connection to have the timestamp it was last POSTed.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/079ba976
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/079ba976
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/079ba976
Branch: refs/heads/two-dot-o-dev
Commit: 079ba9760f3e8d72dbf08610360d089b9ff62d47
Parents: 0c12abb
Author: Todd Nine <tn...@apigee.com>
Authored: Tue Sep 22 16:59:25 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Tue Sep 22 16:59:25 2015 -0600
----------------------------------------------------------------------
.../corepersistence/CpRelationManager.java | 34 ++++++++++++--------
.../service/ConnectionServiceImpl.java | 5 +--
.../service/ConnectionServiceImplTest.java | 4 +--
3 files changed, 26 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/079ba976/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index 1da6f75..21a2ee7 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -31,7 +31,6 @@ import org.slf4j.LoggerFactory;
import org.springframework.util.Assert;
import org.apache.usergrid.corepersistence.asyncevents.AsyncEventService;
-import org.apache.usergrid.corepersistence.pipeline.builder.PipelineBuilderFactory;
import org.apache.usergrid.corepersistence.pipeline.read.ResultsPage;
import org.apache.usergrid.corepersistence.results.ConnectionRefQueryExecutor;
import org.apache.usergrid.corepersistence.results.EntityQueryExecutor;
@@ -62,7 +61,6 @@ import org.apache.usergrid.persistence.graph.Edge;
import org.apache.usergrid.persistence.graph.GraphManager;
import org.apache.usergrid.persistence.graph.SearchByEdge;
import org.apache.usergrid.persistence.graph.SearchByEdgeType;
-import org.apache.usergrid.persistence.graph.impl.SimpleEdge;
import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdge;
import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdgeType;
import org.apache.usergrid.persistence.graph.impl.SimpleSearchEdgeType;
@@ -697,25 +695,35 @@ public class CpRelationManager implements RelationManager {
final GraphManager gm = managerCache.getGraphManager( applicationScope );
- //check if the edge exists
+ //write new edge
+ gm.writeEdge( edge ).subscribe();
- final SearchByEdge searchByEdge = new SimpleSearchByEdge(edge.getSourceNode(), edge.getType(), edge.getTargetNode(), Long.MAX_VALUE, SearchByEdgeType.Order.DESCENDING, Optional.absent() );
+ indexService.queueNewEdge( applicationScope, targetEntity, edge );
- //only take 1 and count it. If we don't have anything, create the edge
- final int count = gm.loadEdgeVersions( searchByEdge ).take( 1 ).count().toBlocking().last();
+ //now read all older versions of an edge, and remove them. Finally calling delete
+ final SearchByEdge searchByEdge =
+ new SimpleSearchByEdge( edge.getSourceNode(), edge.getType(), edge.getTargetNode(), Long.MAX_VALUE,
+ SearchByEdgeType.Order.DESCENDING, Optional.absent() );
- if(count == 0) {
- if(logger.isDebugEnabled()) {
- logger.debug( "No edge exists between {} and {} of type {}. Creating",
- new Object[] { edge.getSourceNode(), edge.getTargetNode(), edge.getType() } );
+
+ //load our versions, only retain the most recent one
+ gm.loadEdgeVersions( searchByEdge ).skip( 1 ).flatMap( edgeToDelete -> {
+ if ( logger.isDebugEnabled() ) {
+ logger.debug( "Marking edge {} for deletion", edgeToDelete );
+ }
+ return gm.markEdge( edge );
+ } ).lastOrDefault( null ).doOnNext( lastEdge -> {
+ //no op if we hit our default
+ if(lastEdge == null){
+ return;
}
- gm.writeEdge( edge ).toBlocking().last();
+ //queue up async processing
+ indexService.queueDeleteEdge( applicationScope, lastEdge );
+ }).subscribe();
- indexService.queueNewEdge( applicationScope, targetEntity, edge );
- }
return connection;
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/079ba976/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/ConnectionServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/ConnectionServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/ConnectionServiceImpl.java
index 1b36321..927d292 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/ConnectionServiceImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/ConnectionServiceImpl.java
@@ -181,9 +181,10 @@ public class ConnectionServiceImpl implements ConnectionService {
logger.debug( "Found edge {}, searching for multiple versions of edge", edge );
+ //keep only the most recent
final SearchByEdge searchByEdge =
- new SimpleSearchByEdge( edge.getSourceNode(), edge.getType(), edge.getTargetNode(), 0,
- SearchByEdgeType.Order.ASCENDING, Optional.absent() );
+ new SimpleSearchByEdge( edge.getSourceNode(), edge.getType(), edge.getTargetNode(), Long.MAX_VALUE,
+ SearchByEdgeType.Order.DESCENDING, Optional.absent() );
return gm.loadEdgeVersions( searchByEdge )
//skip the first version since it's the one we want to retain
.skip( 1 )
http://git-wip-us.apache.org/repos/asf/usergrid/blob/079ba976/stack/core/src/test/java/org/apache/usergrid/corepersistence/service/ConnectionServiceImplTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/service/ConnectionServiceImplTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/service/ConnectionServiceImplTest.java
index a2c18b1..326e128 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/service/ConnectionServiceImplTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/service/ConnectionServiceImplTest.java
@@ -196,7 +196,7 @@ public class ConnectionServiceImplTest {
//check our oldest was deleted first
assertEdgeData( written2, deletedConnections.get( 0 ).getEdge() );
- assertEdgeData( written3, deletedConnections.get( 1 ).getEdge() );
+ assertEdgeData( written1, deletedConnections.get( 1 ).getEdge() );
assertEquals( "2 edges deleted", 2, deletedConnections.size() );
@@ -213,7 +213,7 @@ public class ConnectionServiceImplTest {
assertEquals( 1, edges.size() );
- assertEquals( written1, edges.get( 0 ) );
+ assertEquals( written3, edges.get( 0 ) );
}
[3/6] usergrid git commit: Fixed bugs in migration of apps
Posted by to...@apache.org.
Fixed bugs in migration of apps
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/521fc03e
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/521fc03e
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/521fc03e
Branch: refs/heads/two-dot-o-dev
Commit: 521fc03e556af47d110f0a7d98f2af215cb3fda1
Parents: 8281430
Author: Todd Nine <tn...@apigee.com>
Authored: Thu Sep 24 11:07:24 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Thu Sep 24 11:07:24 2015 -0600
----------------------------------------------------------------------
.../management/AppInfoMigrationPlugin.java | 47 +++++++++++---------
1 file changed, 25 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/521fc03e/stack/services/src/main/java/org/apache/usergrid/management/AppInfoMigrationPlugin.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/AppInfoMigrationPlugin.java b/stack/services/src/main/java/org/apache/usergrid/management/AppInfoMigrationPlugin.java
index 54779b9..d0f39e9 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/AppInfoMigrationPlugin.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/AppInfoMigrationPlugin.java
@@ -164,10 +164,18 @@ public class AppInfoMigrationPlugin implements MigrationPlugin {
throw new RuntimeException( e );
}
}
- } ).doOnCompleted( () -> {
- migrationInfoSerialization.setVersion( getName(), getMaxVersion() );
- observer.complete();
- } ).toBlocking().lastOrDefault( null );
+ } )
+ //we want a doOnError to catch something going wrong, otherwise we'll mark as complete
+ .doOnError( error -> {
+ logger.error( "Unable to migrate applications, an error occurred. Please try again", error );
+ observer.failed( getMaxVersion(), "Unable to migrate applications", error );
+ } )
+
+ //if we complete successfully, set the version and notify the observer
+ .doOnCompleted( () -> {
+ migrationInfoSerialization.setVersion( getName(), getMaxVersion() );
+ observer.complete();
+ } ).subscribe();
}
@@ -186,28 +194,23 @@ public class AppInfoMigrationPlugin implements MigrationPlugin {
UUID applicationId = getUuid( oldAppInfoMap, "applicationUuid" );
applicationId = applicationId == null ? getUuid( oldAppInfoMap, "appUuid" ) : applicationId;
//get app info from graph to see if it has been migrated already
- org.apache.usergrid.persistence.Entity appInfo = getApplicationInfo( applicationId );
- if ( appInfo == null ) {
- observer.update( getMaxVersion(), "Created application_info for " + appName );
- // create org->app connections, but not for apps in dummy "usergrid" internal organization
- if ( !orgName.equals( "usergrid" ) ) { //avoid management org
- EntityRef orgRef = managementEm.getAlias( Group.ENTITY_TYPE, orgName );
- // create and connect new APPLICATION_INFO oldAppInfo to Organization
- managementService.createApplication( orgRef.getUuid(), name, applicationId, null );
- }
- }
- else {
- //already migrated don't do anything
- observer.update( getMaxVersion(),
- "Received existing application_info for " + appName + " don't do anything" );
- }
+
+ // create org->app connections, but not for apps in dummy "usergrid" internal organization
+ //avoid management org
+
+ EntityRef orgRef = managementEm.getAlias( Group.ENTITY_TYPE, orgName );
+ // create and connect new APPLICATION_INFO oldAppInfo to Organization
+ managementService.createApplication( orgRef.getUuid(), name, applicationId, null );
+
+ observer.update( getMaxVersion(), "Created application_info for " + appName );
+ }
+ //swallow
+ catch ( ApplicationAlreadyExistsException appExists ) {
+ logger.info( "Application {} already migrated. Ignoring.", name );
}
catch ( Exception e ) {
- String msg = "Exception writing application_info for " + name;
- logger.error( msg, e );
- observer.failed( getMaxVersion(), msg );
throw new RuntimeException( e );
}
}
[5/6] usergrid git commit: Updated plugin to also take into account
core data version
Posted by to...@apache.org.
Updated plugin to also take into account core data version
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/0b243c4d
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/0b243c4d
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/0b243c4d
Branch: refs/heads/two-dot-o-dev
Commit: 0b243c4d516e0f6ae58421c63787421b8b806052
Parents: 29c1a12
Author: Todd Nine <tn...@apigee.com>
Authored: Thu Sep 24 14:02:40 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Thu Sep 24 14:02:40 2015 -0600
----------------------------------------------------------------------
stack/scripts/migrate_entity_data.py | 66 ++++++++++++++++++++++---------
1 file changed, 48 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/0b243c4d/stack/scripts/migrate_entity_data.py
----------------------------------------------------------------------
diff --git a/stack/scripts/migrate_entity_data.py b/stack/scripts/migrate_entity_data.py
index 4258a71..36f73c0 100644
--- a/stack/scripts/migrate_entity_data.py
+++ b/stack/scripts/migrate_entity_data.py
@@ -51,9 +51,11 @@ import json
# Version expected in status response post-migration for entity and app-info data
-TARGET_VERSION = 2
+TARGET_APPINFO_VERSION=2
+TARGET_ENTITY_DATA_VERSION=2
+TARGET_CORE_DATA_VERSION=2
TARGET_MIGRATION_SYSTEM_VERSION = 1
-TARGET_INDEX_MAPPING_VERSION = 1
+TARGET_INDEX_MAPPING_VERSION = 2
# Set an interval (in seconds) for checking if re-index and/or migration has finished
STATUS_INTERVAL_SECONDS = 2
@@ -63,6 +65,7 @@ PLUGIN_MIGRATION_SYSTEM = 'migration-system'
PLUGIN_APPINFO = 'appinfo-migration'
PLUGIN_ENTITYDATA = 'collections-entity-data'
PLUGIN_INDEX_MAPPING = 'index_mapping_migration'
+PLUGIN_CORE_DATA = 'core-data'
@@ -144,9 +147,21 @@ class Migrate:
if migration_system_updated:
break
+ index_mapping_updated = self.is_index_mapping_updated()
+
+ if not index_mapping_updated:
+ self.logger.info('Index Mapping needs to be updated. Updating index mapping..')
+ self.start_index_mapping_migration()
+ while not index_mapping_updated:
+ time.sleep(STATUS_INTERVAL_SECONDS)
+ index_mapping_updated = self.is_index_mapping_updated()
+ if index_mapping_updated:
+ break
+
# Run AppInfo migration only when both appinfos and collection entity data have not been migrated
if not self.is_data_migrated():
+ #Migrate app info
if self.is_appinfo_migrated():
self.logger.info('AppInfo already migrated. Resetting version for re-migration.')
self.reset_appinfo_migration()
@@ -164,20 +179,14 @@ class Migrate:
self.metrics['appinfo_migration_end'] = get_current_time()
break
self.logger.info('AppInfo Migration Ended.')
+
+
else:
self.logger.info('Full Data Migration previously ran... skipping AppInfo migration.')
- # We need to check and roll index mapping version to 1 if not already there
- index_mapping_updated = self.is_index_mapping_updated()
- if not index_mapping_updated:
- self.logger.info('Index Mapping needs to be updated. Updating index mapping..')
- self.start_index_mapping_migration()
- while not index_mapping_updated:
- time.sleep(STATUS_INTERVAL_SECONDS)
- index_mapping_updated = self.is_index_mapping_updated()
- if index_mapping_updated:
- break
+
+ # We need to check and roll index mapping version to 1 if not already there
# Perform system re-index (it will grab date from input if provided)
job = self.start_reindex()
@@ -202,7 +211,9 @@ class Migrate:
time.sleep(STATUS_INTERVAL_SECONDS)
self.is_data_migrated()
+ # self.start_core_data_migration()
self.start_fulldata_migration()
+
self.metrics['full_data_migration_start'] = get_current_time()
self.logger.info("Full Data Migration Started")
is_migrated = False
@@ -242,6 +253,21 @@ class Migrate:
url = self.endpoint + '/system/index/rebuild'
return url
+ def get_management_reindex_url(self):
+ url = self.get_reindex_url() + "/management"
+ return url
+
+
+ def start_core_data_migration(self):
+ try:
+ r = requests.put(url=self.get_migration_url(), auth=(self.admin_user, self.admin_pass))
+ response = r.json()
+ return response
+ except requests.exceptions.RequestException as e:
+ self.logger.error('Failed to start migration, %s', e)
+ exit_on_error(str(e))
+
+
def start_fulldata_migration(self):
try:
r = requests.put(url=self.get_migration_url(), auth=(self.admin_user, self.admin_pass))
@@ -253,6 +279,7 @@ class Migrate:
def start_migration_system_update(self):
try:
+ #TODO fix this URL
migrateUrl = self.get_migration_url() + '/' + PLUGIN_MIGRATION_SYSTEM
r = requests.put(url=migrateUrl, auth=(self.admin_user, self.admin_pass))
response = r.json()
@@ -293,7 +320,7 @@ class Migrate:
exit_on_error(str(e))
def reset_data_migration(self):
- version = TARGET_VERSION - 1
+ version = TARGET_ENTITY_DATA_VERSION - 1
body = json.dumps({PLUGIN_ENTITYDATA: version, PLUGIN_APPINFO: version})
try:
r = requests.put(url=self.get_reset_migration_url(), data=body, auth=(self.admin_user, self.admin_pass))
@@ -306,7 +333,7 @@ class Migrate:
exit_on_error(str(e))
def reset_appinfo_migration(self):
- version = TARGET_VERSION - 1
+ version = TARGET_APPINFO_VERSION - 1
body = json.dumps({PLUGIN_APPINFO: version})
try:
r = requests.put(url=self.get_reset_migration_url(), data=body, auth=(self.admin_user, self.admin_pass))
@@ -322,14 +349,17 @@ class Migrate:
if status is not None:
entity_version = status['data'][PLUGIN_ENTITYDATA]
appinfo_version = status['data'][PLUGIN_APPINFO]
+ core_data_version = status['data'][PLUGIN_CORE_DATA]
- if entity_version == TARGET_VERSION and appinfo_version == TARGET_VERSION:
+ if entity_version == TARGET_ENTITY_DATA_VERSION and appinfo_version == TARGET_APPINFO_VERSION and core_data_version == TARGET_CORE_DATA_VERSION:
self.logger.info('Full Data Migration status=[COMPLETE], %s=[%s], '
- '%s=[%s]',
+ '%s=[%s], %s=%s',
PLUGIN_ENTITYDATA,
entity_version,
PLUGIN_APPINFO,
- appinfo_version)
+ appinfo_version,
+ PLUGIN_CORE_DATA,
+ core_data_version)
return True
else:
self.logger.info('Full Data Migration status=[NOTSTARTED/INPROGRESS]')
@@ -340,7 +370,7 @@ class Migrate:
if status is not None:
appinfo_version = status['data'][PLUGIN_APPINFO]
- if appinfo_version == TARGET_VERSION:
+ if appinfo_version == TARGET_APPINFO_VERSION:
self.logger.info('AppInfo Migration status=[COMPLETE],'
'%s=[%s]',
PLUGIN_APPINFO,
[4/6] usergrid git commit: Fixes missing observer update
Posted by to...@apache.org.
Fixes missing observer update
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/29c1a12d
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/29c1a12d
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/29c1a12d
Branch: refs/heads/two-dot-o-dev
Commit: 29c1a12dd1a1f070b1c8523e6213f1bf5ac12931
Parents: 521fc03
Author: Todd Nine <tn...@apigee.com>
Authored: Thu Sep 24 12:45:20 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Thu Sep 24 12:45:20 2015 -0600
----------------------------------------------------------------------
.../management/AppInfoMigrationPlugin.java | 20 +++++---------------
1 file changed, 5 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/29c1a12d/stack/services/src/main/java/org/apache/usergrid/management/AppInfoMigrationPlugin.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/AppInfoMigrationPlugin.java b/stack/services/src/main/java/org/apache/usergrid/management/AppInfoMigrationPlugin.java
index d0f39e9..ff6bbe3 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/AppInfoMigrationPlugin.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/AppInfoMigrationPlugin.java
@@ -150,20 +150,10 @@ public class AppInfoMigrationPlugin implements MigrationPlugin {
//get old app infos to migrate
final Observable<Entity> oldAppInfos = getOldAppInfos();
oldAppInfos.doOnNext( oldAppInfoEntity -> {
- try {
- migrateAppInfo( oldAppInfoEntity, observer );
- count.incrementAndGet();
- }
- catch ( Exception e ) {
- if ( e.getCause() instanceof ApplicationAlreadyExistsException ) {
- count.incrementAndGet();
- logger.info( "Application already migrated", oldAppInfoEntity.getId().getUuid() );
- }
- else {
- logger.error( "Failed to migrate app info" + oldAppInfoEntity.getId().getUuid(), e );
- throw new RuntimeException( e );
- }
- }
+
+
+ migrateAppInfo( oldAppInfoEntity, observer );
+ count.incrementAndGet();
} )
//we want a doOnError to catch something going wrong, otherwise we'll mark as complete
.doOnError( error -> {
@@ -196,7 +186,6 @@ public class AppInfoMigrationPlugin implements MigrationPlugin {
//get app info from graph to see if it has been migrated already
-
// create org->app connections, but not for apps in dummy "usergrid" internal organization
//avoid management org
@@ -209,6 +198,7 @@ public class AppInfoMigrationPlugin implements MigrationPlugin {
//swallow
catch ( ApplicationAlreadyExistsException appExists ) {
logger.info( "Application {} already migrated. Ignoring.", name );
+ observer.update( getMaxVersion(), "Skipping application " + name + " it already exists" );
}
catch ( Exception e ) {
throw new RuntimeException( e );
[2/6] usergrid git commit: Adds rest tests and fixes bugs with
POST/PUT re-ordering
Posted by to...@apache.org.
Adds rest tests and fixes bugs with POST/PUT re-ordering
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/82814303
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/82814303
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/82814303
Branch: refs/heads/two-dot-o-dev
Commit: 82814303135e47cc976fea373396aad85afb07f5
Parents: 079ba97
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Sep 23 11:32:11 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Sep 23 11:32:11 2015 -0600
----------------------------------------------------------------------
.../corepersistence/CpRelationManager.java | 7 +-
.../users/ConnectionResourceTest.java | 302 ++++++++++++++-----
.../services/AbstractConnectionsService.java | 43 ++-
3 files changed, 275 insertions(+), 77 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/82814303/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index 21a2ee7..de687b3 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -713,15 +713,16 @@ public class CpRelationManager implements RelationManager {
if ( logger.isDebugEnabled() ) {
logger.debug( "Marking edge {} for deletion", edgeToDelete );
}
- return gm.markEdge( edge );
+ return gm.markEdge( edgeToDelete );
} ).lastOrDefault( null ).doOnNext( lastEdge -> {
//no op if we hit our default
if(lastEdge == null){
return;
}
- //queue up async processing
- indexService.queueDeleteEdge( applicationScope, lastEdge );
+ //don't queue delete b/c that de-indexes, we need to delete the edges only since we have a version still existing to index.
+
+ gm.deleteEdge( lastEdge ).subscribe();
}).subscribe();
http://git-wip-us.apache.org/repos/asf/usergrid/blob/82814303/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/ConnectionResourceTest.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/ConnectionResourceTest.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/ConnectionResourceTest.java
index 22a6165..777b04e 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/ConnectionResourceTest.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/ConnectionResourceTest.java
@@ -17,19 +17,26 @@
package org.apache.usergrid.rest.applications.collection.users;
-import com.sun.jersey.api.client.UniformInterfaceException;
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import org.apache.usergrid.rest.test.resource.AbstractRestIT;
import org.apache.usergrid.rest.test.resource.endpoints.CollectionEndpoint;
+import org.apache.usergrid.rest.test.resource.model.ApiResponse;
import org.apache.usergrid.rest.test.resource.model.Collection;
import org.apache.usergrid.rest.test.resource.model.Entity;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import org.apache.usergrid.rest.test.resource.model.QueryParameters;
-import java.io.IOException;
-import java.util.Map;
+import com.sun.jersey.api.client.UniformInterfaceException;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.fail;
/**
@@ -39,44 +46,46 @@ import static org.junit.Assert.*;
* @since 4.0
*/
public class ConnectionResourceTest extends AbstractRestIT {
- private static Logger log = LoggerFactory.getLogger(ConnectionResourceTest.class);
+ private static Logger log = LoggerFactory.getLogger( ConnectionResourceTest.class );
+
@Test
public void connectionsQueryTest() throws IOException {
//create a peep
Entity peep = new Entity();
- peep.put("type", "chicken");
+ peep.put( "type", "chicken" );
- peep = this.app().collection("peeps").post(peep);
+ peep = this.app().collection( "peeps" ).post( peep );
Entity todd = new Entity();
- todd.put("username", "todd");
- todd = this.app().collection("users").post(todd);
+ todd.put( "username", "todd" );
+ todd = this.app().collection( "users" ).post( todd );
Entity scott = new Entity();
- scott.put("username", "scott");
- scott = this.app().collection("users").post(scott);
+ scott.put( "username", "scott" );
+ scott = this.app().collection( "users" ).post( scott );
Entity objectOfDesire = new Entity();
- objectOfDesire.put("codingmunchies", "doritoes");
- objectOfDesire = this.app().collection("snacks").post(objectOfDesire);
+ objectOfDesire.put( "codingmunchies", "doritoes" );
+ objectOfDesire = this.app().collection( "snacks" ).post( objectOfDesire );
refreshIndex();
- Entity toddWant = this.app().collection("users").entity(todd).collection("likes").collection("snacks").entity(objectOfDesire).post();
- assertNotNull(toddWant);
+ Entity toddWant = this.app().collection( "users" ).entity( todd ).collection( "likes" ).collection( "snacks" )
+ .entity( objectOfDesire ).post();
+ assertNotNull( toddWant );
try {
- this.app().collection("users").entity(scott).collection("likes").collection("peeps").entity(peep).get();
- fail("This should throw an exception");
- } catch (UniformInterfaceException uie) {
+ this.app().collection( "users" ).entity( scott ).collection( "likes" ).collection( "peeps" ).entity( peep )
+ .get();
+ fail( "This should throw an exception" );
+ }
+ catch ( UniformInterfaceException uie ) {
// Should return a 404 Not Found
- assertEquals(404, uie.getResponse().getStatus());
+ assertEquals( 404, uie.getResponse().getStatus() );
}
-
-
}
@@ -85,125 +94,274 @@ public class ConnectionResourceTest extends AbstractRestIT {
// create entities thing1 and thing2
Entity thing1 = new Entity();
- thing1.put("name", "thing1");
- thing1 = this.app().collection("things").post(thing1);
+ thing1.put( "name", "thing1" );
+ thing1 = this.app().collection( "things" ).post( thing1 );
Entity thing2 = new Entity();
- thing2.put("name", "thing2");
- thing2 = this.app().collection("things").post(thing2);
+ thing2.put( "name", "thing2" );
+ thing2 = this.app().collection( "things" ).post( thing2 );
refreshIndex();
//create the connection: thing1 likes thing2
- this.app().collection("things").entity(thing1).connection("likes").collection("things").entity(thing2).post();
+ this.app().collection( "things" ).entity( thing1 ).connection( "likes" ).collection( "things" ).entity( thing2 )
+ .post();
refreshIndex();
//test we have the "likes" in our connection meta data response
- thing1 = this.app().collection("things").entity(thing1).get();
+ thing1 = this.app().collection( "things" ).entity( thing1 ).get();
//TODO this is ugly. revisit.
- String url = (String) ((Map<String, Object>) ((Map<String, Object>) thing1.get("metadata")).get("connections")).get("likes");
- assertNotNull("Connection url returned with entity", url);
+ String url = ( String ) ( ( Map<String, Object> ) ( ( Map<String, Object> ) thing1.get( "metadata" ) )
+ .get( "connections" ) ).get( "likes" );
+ assertNotNull( "Connection url returned with entity", url );
//now that we know the URl is correct, follow it
- CollectionEndpoint likesEndpoint = new CollectionEndpoint(url, this.context(), this.app());
+ CollectionEndpoint likesEndpoint = new CollectionEndpoint( url, this.context(), this.app() );
Collection likes = likesEndpoint.get();
- assertNotNull(likes);
+ assertNotNull( likes );
Entity likedEntity = likes.next();
- assertNotNull(likedEntity);
+ assertNotNull( likedEntity );
//make sure the returned entity is thing2
- assertEquals(thing2.getUuid(), likedEntity.getUuid());
+ assertEquals( thing2.getUuid(), likedEntity.getUuid() );
//now follow the loopback, which should be pointers to the other entity
- thing2 = this.app().collection("things").entity(thing2).get();
+ thing2 = this.app().collection( "things" ).entity( thing2 ).get();
//TODO this is ugly. revisit.
- url = (String) ((Map<String, Object>) ((Map<String, Object>) thing2.get("metadata")).get("connecting")).get("likes");
- assertNotNull("Connecting url returned with entity", url);
+ url = ( String ) ( ( Map<String, Object> ) ( ( Map<String, Object> ) thing2.get( "metadata" ) )
+ .get( "connecting" ) ).get( "likes" );
+ assertNotNull( "Connecting url returned with entity", url );
- CollectionEndpoint likedByEndpoint = new CollectionEndpoint(url, this.context(), this.app());
+ CollectionEndpoint likedByEndpoint = new CollectionEndpoint( url, this.context(), this.app() );
Collection likedBy = likedByEndpoint.get();
- assertNotNull(likedBy);
+ assertNotNull( likedBy );
Entity likedByEntity = likedBy.next();
- assertNotNull(likedByEntity);
+ assertNotNull( likedByEntity );
//make sure the returned entity is thing1
- assertEquals(thing1.getUuid(), likedByEntity.getUuid());
-
+ assertEquals( thing1.getUuid(), likedByEntity.getUuid() );
}
/**
- * Ensure that the connected entity can be deleted
- * properly after it has been connected to another entity
- *
- * @throws IOException
+ * Ensure that the connected entity can be deleted properly after it has been connected to another entity
*/
@Test //USERGRID-3011
public void connectionsDeleteSecondEntityInConnectionTest() throws IOException {
//Create 2 entities, thing1 and thing2
Entity thing1 = new Entity();
- thing1.put("name", "thing1");
- thing1 = this.app().collection("things").post(thing1);
+ thing1.put( "name", "thing1" );
+ thing1 = this.app().collection( "things" ).post( thing1 );
Entity thing2 = new Entity();
- thing2.put("name", "thing2");
- thing2 = this.app().collection("things").post(thing2);
+ thing2.put( "name", "thing2" );
+ thing2 = this.app().collection( "things" ).post( thing2 );
refreshIndex();
//create the connection: thing1 likes thing2
- this.app().collection("things").entity(thing1).connection("likes").collection("things").entity(thing2).post();
+ this.app().collection( "things" ).entity( thing1 ).connection( "likes" ).collection( "things" ).entity( thing2 )
+ .post();
//delete thing2
- this.app().collection("things").entity(thing2).delete();
+ this.app().collection( "things" ).entity( thing2 ).delete();
refreshIndex();
try {
//attempt to retrieve thing1
- thing2 = this.app().collection("things").entity(thing2).get();
- fail("This should throw an exception");
- } catch (UniformInterfaceException uie) {
+ thing2 = this.app().collection( "things" ).entity( thing2 ).get();
+ fail( "This should throw an exception" );
+ }
+ catch ( UniformInterfaceException uie ) {
// Should return a 404 Not Found
- assertEquals(404, uie.getResponse().getStatus());
+ assertEquals( 404, uie.getResponse().getStatus() );
}
}
+
/**
- * Ensure that the connecting entity can be deleted
- * properly after a connection has been added
- *
- * @throws IOException
+ * Ensure that the connecting entity can be deleted properly after a connection has been added
*/
@Test //USERGRID-3011
public void connectionsDeleteFirstEntityInConnectionTest() throws IOException {
//Create 2 entities, thing1 and thing2
Entity thing1 = new Entity();
- thing1.put("name", "thing1");
- thing1 = this.app().collection("things").post(thing1);
+ thing1.put( "name", "thing1" );
+ thing1 = this.app().collection( "things" ).post( thing1 );
Entity thing2 = new Entity();
- thing2.put("name", "thing2");
- thing2 = this.app().collection("things").post(thing2);
+ thing2.put( "name", "thing2" );
+ thing2 = this.app().collection( "things" ).post( thing2 );
refreshIndex();
//create the connection: thing1 likes thing2
- this.app().collection("things").entity(thing1).connection("likes").collection("things").entity(thing2).post();
+ this.app().collection( "things" ).entity( thing1 ).connection( "likes" ).collection( "things" ).entity( thing2 )
+ .post();
//delete thing1
- this.app().collection("things").entity(thing1).delete();
+ this.app().collection( "things" ).entity( thing1 ).delete();
refreshIndex();
try {
//attempt to retrieve thing1
- thing1 = this.app().collection("things").entity(thing1).get();
- fail("This should throw an exception");
- } catch (UniformInterfaceException uie) {
+ thing1 = this.app().collection( "things" ).entity( thing1 ).get();
+ fail( "This should throw an exception" );
+ }
+ catch ( UniformInterfaceException uie ) {
// Should return a 404 Not Found
- assertEquals(404, uie.getResponse().getStatus());
+ assertEquals( 404, uie.getResponse().getStatus() );
}
+ }
+
+
+ /**
+ * UERGRID-1018
+ */
+ @Test
+ public void testRePostOrder() {
+
+ Entity thing1 = new Entity();
+ thing1.put( "name", "thing1" );
+
+ final CollectionEndpoint collection = this.app().collection( "things" );
+
+ thing1 = collection.post( thing1 );
+
+ Entity thing2 = new Entity();
+ thing2.put( "name", "thing2" );
+ thing2 = collection.post( thing2 );
+
+ Entity thing3 = new Entity();
+ thing3.put( "name", "thing3" );
+ thing3 = collection.post( thing3 );
+
+ //now connect them
+
+ //connect thing1 -> thing2
+
+ final CollectionEndpoint connectionEndpoint = collection.entity( thing1 ).connection( "connectorder" );
+ connectionEndpoint.entity( thing2 ).post();
+ //connect thing1 -> thing3
+ connectionEndpoint.entity( thing3 ).post();
+
+ refreshIndex();
+
+ //now do a GET, we should see thing2 then thing3
+
+ final ApiResponse order1 = connectionEndpoint.get().getResponse();
+
+ //now verify order
+ verifyOrder( order1, thing3, thing2 );
+
+ final QueryParameters queryParameters = new QueryParameters();
+ queryParameters.setQuery( "select * order by modified desc" );
+
+ final ApiResponse order1Query = connectionEndpoint.get( queryParameters ).getResponse();
+
+ //now verify order
+ verifyOrder( order1Query, thing3, thing2 );
+
+
+ //now re-post thing 2 it should appear second
+ connectionEndpoint.entity( thing2 ).post();
+
+ refreshIndex();
+
+
+ final ApiResponse order2 = connectionEndpoint.get().getResponse();
+
+ //now verify order
+ verifyOrder( order2, thing2, thing3 );
+
+ final ApiResponse order2Query = connectionEndpoint.get( queryParameters ).getResponse();
+
+ //now verify order
+ verifyOrder( order2Query, thing3, thing2 );
}
+ /**
+ * UERGRID-1018
+ */
+ @Test
+ public void testRePutOrder() {
+
+ Entity thing1 = new Entity();
+ thing1.put( "name", "thing1" );
+
+ final CollectionEndpoint collection = this.app().collection( "things" );
+
+ thing1 = collection.post( thing1 );
+
+ Entity thing2 = new Entity();
+ thing2.put( "name", "thing2" );
+ thing2 = collection.post( thing2 );
+
+
+ Entity thing3 = new Entity();
+ thing3.put( "name", "thing3" );
+ thing3 = collection.post( thing3 );
+
+ //now connect them
+
+ //connect thing1 -> thing2
+
+ final CollectionEndpoint connectionEndpoint = collection.entity( thing1 ).connection( "connectorder" );
+ connectionEndpoint.entity( thing2 ).put( thing2 );
+
+ //connect thing1 -> thing3
+ connectionEndpoint.entity( thing3 ).put( thing3 );
+
+ refreshIndex();
+
+ //now do a GET, we should see thing2 then thing3
+
+ final ApiResponse order1 = connectionEndpoint.get().getResponse();
+
+ //now verify order
+ verifyOrder( order1, thing3, thing2 );
+
+ final QueryParameters queryParameters = new QueryParameters();
+ queryParameters.setQuery( "select * order by modified desc" );
+
+ final ApiResponse order1Query = connectionEndpoint.get( queryParameters ).getResponse();
+
+ //now verify order
+ verifyOrder( order1Query, thing3, thing2 );
+
+
+ //now re-post thing 2 it should appear second
+ connectionEndpoint.entity( thing2 ).put( thing2 );
+
+ refreshIndex();
+
+ final ApiResponse order2 = connectionEndpoint.get().getResponse();
+
+
+ //now verify order
+ verifyOrder( order2, thing2, thing3 );
+ }
+
+
+ /**
+ * Verify our response
+ */
+ private void verifyOrder( final ApiResponse apiResponse, final Entity... verifyOrder ) {
+
+ final List<Entity> responseEntities = apiResponse.getEntities();
+
+ assertEquals( "Size should be equals", verifyOrder.length, responseEntities.size() );
+
+
+ for ( int i = 0; i < verifyOrder.length; i++ ) {
+
+ final Entity verifyEntity = verifyOrder[i];
+
+ final Entity returned = responseEntities.get( i );
+
+ assertEquals( "Should be the same entity", verifyEntity.getUuid(), returned.getUuid() );
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/82814303/stack/services/src/main/java/org/apache/usergrid/services/AbstractConnectionsService.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/AbstractConnectionsService.java b/stack/services/src/main/java/org/apache/usergrid/services/AbstractConnectionsService.java
index c4142a0..ee322d2 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/AbstractConnectionsService.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/AbstractConnectionsService.java
@@ -23,20 +23,25 @@ import java.util.UUID;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
import org.apache.usergrid.persistence.ConnectionRef;
import org.apache.usergrid.persistence.Entity;
import org.apache.usergrid.persistence.EntityRef;
-import org.apache.usergrid.persistence.index.query.Identifier;
import org.apache.usergrid.persistence.Query;
+import org.apache.usergrid.persistence.Query.Level;
import org.apache.usergrid.persistence.Results;
import org.apache.usergrid.persistence.Schema;
import org.apache.usergrid.persistence.SimpleEntityRef;
-import org.apache.usergrid.persistence.Query.Level;
+import org.apache.usergrid.persistence.index.query.Identifier;
import org.apache.usergrid.services.ServiceParameter.IdParameter;
import org.apache.usergrid.services.ServiceParameter.NameParameter;
import org.apache.usergrid.services.ServiceParameter.QueryParameter;
import org.apache.usergrid.services.ServiceResults.Type;
import org.apache.usergrid.services.exceptions.ServiceResourceNotFoundException;
+
+import rx.Observable;
+import rx.schedulers.Schedulers;
+
import static org.apache.usergrid.services.ServiceParameter.filter;
import static org.apache.usergrid.services.ServiceParameter.firstParameterIsName;
import static org.apache.usergrid.utils.ClassUtils.cast;
@@ -398,6 +403,11 @@ public class AbstractConnectionsService extends AbstractService {
String entityType = getEntityType();
item = em.create( id, entityType, context.getPayload().getProperties() );
}
+
+ //create the connection
+ createConnection( context.getOwner(), context.getCollectionName(), item );
+
+
return new ServiceResults( this, context, Type.CONNECTION, Results.fromEntity( item ), null, null );
}
@@ -426,6 +436,35 @@ public class AbstractConnectionsService extends AbstractService {
updateEntities( context, r );
+ //create the connection
+
+ //TODO wire the RX scheduler in here and use our parallelism system
+
+
+ /**
+ * Create all the connections for all the entities
+ */
+ final List<Entity> entities = r.getEntities();
+ if ( entities != null ) {
+
+ /**
+ * Save up to 10 connections in parallel
+ */
+ Observable.from(entities).flatMap( emittedEntity -> {
+ return Observable.just( emittedEntity ).doOnNext( toSave -> {
+ try {
+ createConnection( context.getOwner(), context.getCollectionName(), toSave );
+ }
+ catch ( Exception e ) {
+ throw new RuntimeException( "Unable to save connection", e );
+ }
+ }).subscribeOn( Schedulers.io() );
+ }, 10).subscribe();
+
+
+ }
+
+
return new ServiceResults( this, context, Type.CONNECTION, r, null, null );
}
[6/6] usergrid git commit: Merge branch 'refs/heads/2.1-release' into
two-dot-o-dev
Posted by to...@apache.org.
Merge branch 'refs/heads/2.1-release' into two-dot-o-dev
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/eda816bb
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/eda816bb
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/eda816bb
Branch: refs/heads/two-dot-o-dev
Commit: eda816bb0679b41d2d77b5e19f5d61ba9cb799b6
Parents: 680338c 0b243c4
Author: Todd Nine <tn...@apigee.com>
Authored: Thu Sep 24 14:15:13 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Thu Sep 24 14:15:13 2015 -0600
----------------------------------------------------------------------
.../corepersistence/CpRelationManager.java | 35 ++-
.../service/ConnectionServiceImpl.java | 5 +-
.../service/ConnectionServiceImplTest.java | 4 +-
.../users/ConnectionResourceTest.java | 292 ++++++++++++++-----
stack/scripts/migrate_entity_data.py | 66 +++--
.../management/AppInfoMigrationPlugin.java | 69 ++---
.../services/AbstractConnectionsService.java | 43 ++-
7 files changed, 369 insertions(+), 145 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/eda816bb/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/eda816bb/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/ConnectionResourceTest.java
----------------------------------------------------------------------
diff --cc stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/ConnectionResourceTest.java
index 3655300,777b04e..34cb56e
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/ConnectionResourceTest.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/ConnectionResourceTest.java
@@@ -17,19 -17,26 +17,20 @@@
package org.apache.usergrid.rest.applications.collection.users;
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
++import com.sun.jersey.api.client.UniformInterfaceException;
import org.apache.usergrid.rest.test.resource.AbstractRestIT;
import org.apache.usergrid.rest.test.resource.endpoints.CollectionEndpoint;
+ import org.apache.usergrid.rest.test.resource.model.ApiResponse;
import org.apache.usergrid.rest.test.resource.model.Collection;
import org.apache.usergrid.rest.test.resource.model.Entity;
- import org.junit.Test;
- import org.slf4j.Logger;
- import org.slf4j.LoggerFactory;
+ import org.apache.usergrid.rest.test.resource.model.QueryParameters;
- import javax.ws.rs.ClientErrorException;
-import com.sun.jersey.api.client.UniformInterfaceException;
+import java.io.IOException;
+import java.util.Map;
- import static org.junit.Assert.*;
+ import static org.junit.Assert.assertEquals;
+ import static org.junit.Assert.assertNotNull;
+ import static org.junit.Assert.fail;
/**