You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by gr...@apache.org on 2015/05/26 20:13:00 UTC
[01/22] incubator-usergrid git commit: [USERGRID-572] - ignore
sensitive params from QP in response params
Repository: incubator-usergrid
Updated Branches:
refs/heads/USERGRID-608 aa8b20d9a -> 3f8899dfa
[USERGRID-572] - ignore sensitive params from QP in response params
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/072e2d4f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/072e2d4f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/072e2d4f
Branch: refs/heads/USERGRID-608
Commit: 072e2d4f9dd15c1f859ee110b3997f6e8d79c256
Parents: 472ccaf
Author: Senthil Kumar K <se...@gmail.com>
Authored: Wed Apr 22 11:38:38 2015 -0700
Committer: Senthil Kumar K <se...@gmail.com>
Committed: Wed Apr 22 11:38:38 2015 -0700
----------------------------------------------------------------------
.../org/apache/usergrid/rest/ApiResponse.java | 6 +++
.../apache/usergrid/rest/ApiResponseTest.java | 45 ++++++++++++++++++++
2 files changed, 51 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/072e2d4f/stack/rest/src/main/java/org/apache/usergrid/rest/ApiResponse.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/ApiResponse.java b/stack/rest/src/main/java/org/apache/usergrid/rest/ApiResponse.java
index d7dd5f8..93e7f84 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/ApiResponse.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/ApiResponse.java
@@ -24,6 +24,8 @@ import com.fasterxml.jackson.annotation.JsonPropertyOrder;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import com.fasterxml.jackson.databind.annotation.JsonSerialize.Inclusion;
import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@@ -85,6 +87,9 @@ public class ApiResponse {
protected Map<String, Object> properties = new TreeMap<String, Object>( String.CASE_INSENSITIVE_ORDER );
+ protected final Collection<String> IGNORE_QP = Arrays.asList("client_id", "client_secret", "password", "username", "access_token",
+ "client_credentials", "fb_access_token", "fq_access_token", "ping_access_token", "token");
+
@Autowired
protected ServerEnvironmentProperties serverEnvironmentProperties;
@@ -556,6 +561,7 @@ public class ApiResponse {
public void setParams( Map<String, List<String>> params ) {
Map<String, List<String>> q = new LinkedHashMap<String, List<String>>();
for ( String k : params.keySet() ) {
+ if (IGNORE_QP.contains(k.toLowerCase())) continue;
List<String> v = params.get( k );
if ( v != null ) {
q.put( k, new ArrayList<String>( v ) );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/072e2d4f/stack/rest/src/test/java/org/apache/usergrid/rest/ApiResponseTest.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/ApiResponseTest.java b/stack/rest/src/test/java/org/apache/usergrid/rest/ApiResponseTest.java
new file mode 100644
index 0000000..552feaa
--- /dev/null
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/ApiResponseTest.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.usergrid.rest;
+
+import org.junit.Test;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+public class ApiResponseTest {
+
+ @Test
+ public void testIgnoreQP() {
+ ApiResponse apiResponse = new ApiResponse();
+ Map<String, List<String>> params = new HashMap<String, List<String>>();
+ params.put("access_token", Arrays.asList("YWMtL8AQ-ukcEeS2lHs-P-n8wQAAAU0GaCt_Y0cPWeXMJij4x_fW0w_dTMpUH7I"));
+ params.put("name", Arrays.asList("test"));
+ params.put("username", Arrays.asList("abc"));
+ params.put("password", Arrays.asList("123"));
+ apiResponse.setParams(params);
+ assertNull(apiResponse.getParams().get("password"));
+ assertEquals(apiResponse.getParams().size(), 1);
+ }
+}
[18/22] incubator-usergrid git commit: Fixes exception on missing
keyspace during bootstrap
Posted by gr...@apache.org.
Fixes exception on missing keyspace during bootstrap
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/8fde7c5d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/8fde7c5d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/8fde7c5d
Branch: refs/heads/USERGRID-608
Commit: 8fde7c5dfd7d06c17b0a2c2a3ac1053bf3d97e76
Parents: 5a7f9c0
Author: Todd Nine <tn...@apigee.com>
Authored: Fri May 15 19:38:19 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Fri May 15 19:38:19 2015 -0600
----------------------------------------------------------------------
.../data/MigrationInfoSerializationImpl.java | 4 +-
.../migration/schema/MigrationManagerImpl.java | 14 ++----
.../core/migration/util/AstayanxUtils.java | 49 ++++++++++++++++++++
3 files changed, 56 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8fde7c5d/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/MigrationInfoSerializationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/MigrationInfoSerializationImpl.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/MigrationInfoSerializationImpl.java
index 4a349fd..3def798 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/MigrationInfoSerializationImpl.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/MigrationInfoSerializationImpl.java
@@ -35,6 +35,7 @@ import org.apache.usergrid.persistence.core.astyanax.MultiTennantColumnFamilyDef
import org.apache.usergrid.persistence.core.astyanax.ScopedRowKey;
import org.apache.usergrid.persistence.core.astyanax.ScopedRowKeySerializer;
import org.apache.usergrid.persistence.core.astyanax.StringRowCompositeSerializer;
+import org.apache.usergrid.persistence.core.migration.util.AstayanxUtils;
import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.entity.SimpleId;
@@ -149,7 +150,8 @@ public class MigrationInfoSerializationImpl implements MigrationInfoSerializatio
return 0;
}
catch ( ConnectionException e ) {
- throw new DataMigrationException( "Unable to retrieve status", e );
+ AstayanxUtils.isKeyspaceMissing("Unable to connect to cassandra to retrieve status", e);
+ return 0;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8fde7c5d/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/schema/MigrationManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/schema/MigrationManagerImpl.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/schema/MigrationManagerImpl.java
index 31aa1b3..26351cf 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/schema/MigrationManagerImpl.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/schema/MigrationManagerImpl.java
@@ -28,6 +28,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.usergrid.persistence.core.astyanax.MultiTennantColumnFamilyDefinition;
+import org.apache.usergrid.persistence.core.migration.util.AstayanxUtils;
import com.google.common.collect.ImmutableMap;
import com.google.inject.Inject;
@@ -131,22 +132,15 @@ public class MigrationManagerImpl implements MigrationManager {
try {
keyspaceDefinition = keyspace.describeKeyspace();
- }
- catch ( BadRequestException badRequestException ) {
-
- //check if it's b/c the keyspace is missing, if so
- final String message = badRequestException.getMessage();
-
- boolean missingKeyspace = message.contains( "why:Keyspace" ) && message.contains( "does not exist" );
- if ( !missingKeyspace ) {
- throw badRequestException;
- }
}catch( NotFoundException nfe){
//if we execute this immediately after a drop keyspace in 1.2.x, Cassandra is returning the NFE instead of a BadRequestException
//swallow and log, then continue to create the keyspaces.
logger.info( "Received a NotFoundException when attempting to describe keyspace. It does not exist" );
}
+ catch(Exception e){
+ AstayanxUtils.isKeyspaceMissing("Unable to connect to cassandra", e);
+ }
if ( keyspaceDefinition != null ) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8fde7c5d/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/util/AstayanxUtils.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/util/AstayanxUtils.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/util/AstayanxUtils.java
new file mode 100644
index 0000000..7ae4748
--- /dev/null
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/util/AstayanxUtils.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence.core.migration.util;
+
+
+import com.netflix.astyanax.connectionpool.exceptions.BadRequestException;
+
+
+public class AstayanxUtils {
+
+ /**
+ * Return true if the exception is an instance of a missing keysapce
+ * @param rethrowMessage The message to add to the exception if rethrown
+ * @param cassandraException The exception from cassandar
+ * @return
+ */
+ public static void isKeyspaceMissing(final String rethrowMessage, final Exception cassandraException ) {
+
+ if ( cassandraException instanceof BadRequestException ) {
+
+ //check if it's b/c the keyspace is missing, if so
+ final String message = cassandraException.getMessage();
+
+ //no op, just swallow
+ if(message.contains( "why:Keyspace" ) && message.contains( "does not exist" )){
+ return;
+ };
+ }
+
+ throw new RuntimeException( rethrowMessage, cassandraException );
+ }
+}
[07/22] incubator-usergrid git commit: rename connecting to source
and connected to target
Posted by gr...@apache.org.
rename connecting to source and connected to target
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/5b1dfa1c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/5b1dfa1c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/5b1dfa1c
Branch: refs/heads/USERGRID-608
Commit: 5b1dfa1c8906cdf7e1a484b48f1944239e5b48ec
Parents: ed8ee9a
Author: Shawn Feldman <sf...@apache.org>
Authored: Wed May 13 12:30:59 2015 -0700
Committer: Shawn Feldman <sf...@apache.org>
Committed: Wed May 13 12:30:59 2015 -0700
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 22 ++++++-------
.../corepersistence/CpEntityManagerFactory.java | 8 ++---
.../corepersistence/CpRelationManager.java | 15 ++++-----
.../corepersistence/index/IndexServiceImpl.java | 6 ++--
.../usergrid/persistence/EntityManager.java | 34 ++++++++++----------
.../persistence/MultiQueryIterator.java | 4 +--
.../apache/usergrid/persistence/PathQuery.java | 3 +-
.../usergrid/persistence/RelationManager.java | 12 +++----
.../usergrid/persistence/CollectionIT.java | 2 +-
.../usergrid/persistence/CountingMutatorIT.java | 6 ++--
.../persistence/EntityConnectionsIT.java | 26 +++++++--------
.../org/apache/usergrid/persistence/GeoIT.java | 8 ++---
.../PerformanceEntityRebuildIndexTest.java | 8 ++---
.../persistence/query/ConnectionHelper.java | 2 +-
.../persistence/query/IteratingQueryIT.java | 2 +-
.../rest/management/ImportResourceIT.java | 2 +-
.../cassandra/ManagementServiceImpl.java | 12 +++----
.../management/export/ExportServiceImpl.java | 4 +--
.../management/importer/FileImportJob.java | 5 ++-
.../management/importer/ImportServiceImpl.java | 15 ++++-----
.../services/AbstractConnectionsService.java | 24 +++++++-------
.../users/activities/ActivitiesService.java | 4 +--
.../users/activities/ActivitiesService.java | 4 +--
.../management/importer/ImportCollectionIT.java | 2 +-
.../importer/ImportConnectionsTest.java | 9 +++---
.../management/importer/ImportServiceIT.java | 6 ++--
26 files changed, 114 insertions(+), 131 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5b1dfa1c/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 9430c4e..63018cb 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -1515,35 +1515,35 @@ public class CpEntityManager implements EntityManager {
@Override
- public Results getConnectedEntities( EntityRef entityRef, String connectionType,
- String connectedEntityType, Level resultsLevel ) throws Exception {
+ public Results getTargetEntities(EntityRef entityRef, String connectionType,
+ String connectedEntityType, Level resultsLevel) throws Exception {
return getRelationManager( entityRef )
- .getConnectedEntities( connectionType, connectedEntityType, resultsLevel );
+ .getTargetEntities(connectionType, connectedEntityType, resultsLevel);
}
@Override
- public Results getConnectingEntities( EntityRef entityRef, String connectionType,
- String connectedEntityType, Level resultsLevel ) throws Exception {
+ public Results getSourceEntities(EntityRef entityRef, String connectionType,
+ String connectedEntityType, Level resultsLevel) throws Exception {
return getRelationManager( entityRef )
- .getConnectingEntities( connectionType, connectedEntityType, resultsLevel );
+ .getSourceEntities(connectionType, connectedEntityType, resultsLevel);
}
@Override
- public Results getConnectingEntities( EntityRef entityRef, String connectionType,
- String entityType, Level level, int count ) throws Exception {
+ public Results getSourceEntities(EntityRef entityRef, String connectionType,
+ String entityType, Level level, int count) throws Exception {
- return getRelationManager( entityRef ).getConnectingEntities( connectionType, entityType, level, count );
+ return getRelationManager( entityRef ).getSourceEntities(connectionType, entityType, level, count);
}
@Override
- public Results searchConnectedEntities( EntityRef connectingEntity, Query query ) throws Exception {
+ public Results searchTargetEntities(EntityRef connectingEntity, Query query) throws Exception {
- return getRelationManager( connectingEntity ).searchConnectedEntities( query );
+ return getRelationManager( connectingEntity ).searchTargetEntities(query);
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5b1dfa1c/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index e796545..63e2869 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -204,7 +204,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
@Override
public Entity createApplicationV2(String organizationName, String name) throws Exception {
- return createApplicationV2( organizationName, name, null );
+ return createApplicationV2(organizationName, name, null);
}
@@ -262,7 +262,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
properties = new TreeMap<>( CASE_INSENSITIVE_ORDER );
}
properties.put( PROPERTY_NAME, appName );
- EntityManager appEm = getEntityManager( applicationId);
+ EntityManager appEm = getEntityManager(applicationId);
appEm.create(applicationId, TYPE_APPLICATION, properties);
appEm.resetRoles();
// entityIndex.refreshAsync();//.toBlocking().last();
@@ -356,7 +356,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
copyConnections = Observable.from(connectionTypes).doOnNext(connType -> {
try {
final Results connResults =
- managementEm.getConnectedEntities(oldAppEntity, connType, null, Query.Level.ALL_PROPERTIES);
+ managementEm.getTargetEntities(oldAppEntity, connType, null, Query.Level.ALL_PROPERTIES);
connResults.getEntities().forEach(entity -> {
try {
managementEm.createConnection(newAppEntity, connType, entity);
@@ -420,7 +420,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
Map<String, UUID> appMap = new HashMap<>();
ApplicationScope appScope =
- CpNamingUtils.getApplicationScope(CpNamingUtils.MANAGEMENT_APPLICATION_ID );
+ CpNamingUtils.getApplicationScope(CpNamingUtils.MANAGEMENT_APPLICATION_ID);
GraphManager gm = managerCache.getGraphManager(appScope);
EntityManager em = getEntityManager(CpNamingUtils.MANAGEMENT_APPLICATION_ID);
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5b1dfa1c/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index bd3d137..4993d88 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -20,7 +20,6 @@ package org.apache.usergrid.corepersistence;
import java.util.*;
import org.apache.usergrid.persistence.graph.*;
-import org.apache.usergrid.utils.UUIDUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.Assert;
@@ -576,7 +575,7 @@ public class CpRelationManager implements RelationManager {
results = em.getCollection( headEntity, srcRelationName, null, 5000, Level.REFS, false );
}
else {
- results = em.getConnectedEntities( headEntity, srcRelationName, null, Level.REFS );
+ results = em.getTargetEntities(headEntity, srcRelationName, null, Level.REFS);
}
if ( ( results != null ) && ( results.size() > 0 ) ) {
@@ -837,7 +836,7 @@ public class CpRelationManager implements RelationManager {
@Override
- public Results getConnectedEntities( String connectionType, String connectedEntityType, Level level )
+ public Results getTargetEntities(String connectionType, String connectedEntityType, Level level)
throws Exception {
//until this is refactored properly, we will delegate to a search by query
@@ -850,20 +849,20 @@ public class CpRelationManager implements RelationManager {
query.setEntityType( connectedEntityType );
query.setResultsLevel( level );
- return searchConnectedEntities( query );
+ return searchTargetEntities(query);
}
@Override
- public Results getConnectingEntities( String connType, String fromEntityType, Level resultsLevel )
+ public Results getSourceEntities(String connType, String fromEntityType, Level resultsLevel)
throws Exception {
- return getConnectingEntities( connType, fromEntityType, resultsLevel, -1 );
+ return getSourceEntities(connType, fromEntityType, resultsLevel, -1);
}
@Override
- public Results getConnectingEntities( String connType, String fromEntityType, Level level, int count )
+ public Results getSourceEntities(String connType, String fromEntityType, Level level, int count)
throws Exception {
// looking for edges to the head entity
@@ -896,7 +895,7 @@ public class CpRelationManager implements RelationManager {
@Override
- public Results searchConnectedEntities( Query query ) throws Exception {
+ public Results searchTargetEntities(Query query) throws Exception {
Preconditions.checkNotNull( query, "query cannot be null" );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5b1dfa1c/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
index 8185b4d..1fc8eea 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
@@ -104,11 +104,11 @@ public class IndexServiceImpl implements IndexService {
//we might or might not need to index from target-> source
- final Observable<IndexEdge> targetSizes = getIndexEdgesAsTarget( gm, entityId );
+ final Observable<IndexEdge> edgesToSource = getIndexEdgesAsTarget( gm, entityId );
//merge the edges together
- final Observable<IndexEdge> observable = Observable.merge( sourceEdgesToIndex, targetSizes);
+ final Observable<IndexEdge> observable = Observable.merge( sourceEdgesToIndex, edgesToSource);
//do our observable for batching
//try to send a whole batch if we can
@@ -134,8 +134,6 @@ public class IndexServiceImpl implements IndexService {
@Override
public Observable<IndexOperationMessage> indexEdge( final ApplicationScope applicationScope, final Entity entity, final Edge edge ) {
-
-
final Observable<IndexOperationMessage> batches = Observable.just( edge ).map( observableEdge -> {
//if the node is the target node, generate our scope correctly
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5b1dfa1c/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
index b8d3360..774a0fb 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
@@ -420,25 +420,25 @@ public interface EntityManager {
*/
public ConnectionRef createConnection( ConnectionRef connection ) throws Exception;
- public ConnectionRef createConnection( EntityRef connectingEntity, String connectionType,
- EntityRef connectedEntityRef ) throws Exception;
+ public ConnectionRef createConnection( EntityRef sourceEntity, String connectionType,
+ EntityRef targetEntityRef ) throws Exception;
- public ConnectionRef createConnection( EntityRef connectingEntity, String pairedConnectionType,
+ public ConnectionRef createConnection( EntityRef sourceEntity, String pairedConnectionType,
EntityRef pairedEntity, String connectionType,
- EntityRef connectedEntityRef ) throws Exception;
+ EntityRef targetEntityRef ) throws Exception;
public ConnectionRef createConnection(
- EntityRef connectingEntity, ConnectedEntityRef... connections )
+ EntityRef sourceEntity, ConnectedEntityRef... connections )
throws Exception;
- public ConnectionRef connectionRef( EntityRef connectingEntity, String connectionType,
- EntityRef connectedEntityRef ) throws Exception;
+ public ConnectionRef connectionRef( EntityRef sourceEntity, String connectionType,
+ EntityRef targetEntityRef ) throws Exception;
- public ConnectionRef connectionRef( EntityRef connectingEntity, String pairedConnectionType,
- EntityRef pairedEntity, String connectionType, EntityRef connectedEntityRef )
+ public ConnectionRef connectionRef( EntityRef sourceEntity, String pairedConnectionType,
+ EntityRef pairedEntity, String connectionType, EntityRef targetEntityRef )
throws Exception;
- public ConnectionRef connectionRef( EntityRef connectingEntity, ConnectedEntityRef... connections );
+ public ConnectionRef connectionRef( EntityRef sourceEntity, ConnectedEntityRef... connections );
/**
* Disconnects two connected entities with the specified connection type. Connections are
@@ -464,8 +464,8 @@ public interface EntityManager {
*
* @throws Exception the exception
*/
- public Results getConnectedEntities( EntityRef entityRef, String connectionType,
- String connectedEntityType, Level resultsLevel ) throws Exception;
+ public Results getTargetEntities(EntityRef entityRef, String connectionType,
+ String connectedEntityType, Level resultsLevel) throws Exception;
/**
* Gets the entities connecting to this entity, optionally with the specified connection
@@ -481,13 +481,13 @@ public interface EntityManager {
*
* @throws Exception the exception
*/
- public Results getConnectingEntities( EntityRef entityRef, String connectionType,
- String connectedEntityType, Level resultsLevel ) throws Exception;
+ public Results getSourceEntities(EntityRef entityRef, String connectionType,
+ String connectedEntityType, Level resultsLevel) throws Exception;
- public Results getConnectingEntities( EntityRef entityRef, String connectionType,
- String entityType, Level level, int count) throws Exception;
+ public Results getSourceEntities(EntityRef entityRef, String connectionType,
+ String entityType, Level level, int count) throws Exception;
- public Results searchConnectedEntities( EntityRef connectingEntity, Query query ) throws Exception;
+ public Results searchTargetEntities(EntityRef sourceEntity, Query query) throws Exception;
// Application roles
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5b1dfa1c/stack/core/src/main/java/org/apache/usergrid/persistence/MultiQueryIterator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/MultiQueryIterator.java b/stack/core/src/main/java/org/apache/usergrid/persistence/MultiQueryIterator.java
index c32f429..3662c63 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/MultiQueryIterator.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/MultiQueryIterator.java
@@ -17,10 +17,8 @@
package org.apache.usergrid.persistence;
-import org.apache.usergrid.persistence.Query;
import java.util.Iterator;
import java.util.UUID;
-import org.apache.usergrid.persistence.Query.Level;
/**
@@ -94,7 +92,7 @@ public class MultiQueryIterator implements ResultsIterator {
try {
return ( query.getCollection() != null ) ?
entityManager.searchCollection( entityRef, query.getCollection(), query ) :
- entityManager.searchConnectedEntities( entityRef, query );
+ entityManager.searchTargetEntities(entityRef, query);
}
catch ( Exception e ) {
throw new RuntimeException( e );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5b1dfa1c/stack/core/src/main/java/org/apache/usergrid/persistence/PathQuery.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/PathQuery.java b/stack/core/src/main/java/org/apache/usergrid/persistence/PathQuery.java
index 58b9522..bb336e1 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/PathQuery.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/PathQuery.java
@@ -17,7 +17,6 @@
package org.apache.usergrid.persistence;
-import org.apache.usergrid.persistence.Query;
import java.util.Iterator;
import java.util.UUID;
@@ -103,7 +102,7 @@ public class PathQuery<E> {
EntityRef ref = new SimpleEntityRef(type,uuid);
return ( query.getCollection() != null ) ?
em.searchCollection( ref, query.getCollection(), query ) :
- em.searchConnectedEntities( ref, query );
+ em.searchTargetEntities(ref, query);
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5b1dfa1c/stack/core/src/main/java/org/apache/usergrid/persistence/RelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/RelationManager.java b/stack/core/src/main/java/org/apache/usergrid/persistence/RelationManager.java
index 84e4a38..0b2fca8 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/RelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/RelationManager.java
@@ -106,11 +106,11 @@ public interface RelationManager {
* @param connectionType The type/name of the connection
* @param connectedEntityType The type of
*/
- public Results getConnectedEntities( String connectionType, String connectedEntityType, Level resultsLevel )
+ public Results getTargetEntities(String connectionType, String connectedEntityType, Level resultsLevel)
throws Exception;
- public Results getConnectingEntities( String connectionType, String connectedEntityType,
- Level resultsLevel ) throws Exception;
+ public Results getSourceEntities(String connectionType, String connectedEntityType,
+ Level resultsLevel) throws Exception;
// public Results searchConnectedEntitiesForProperty(String connectionType,
// String connectedEntityType, String propertyName,
@@ -118,10 +118,10 @@ public interface RelationManager {
// UUID startResult, int count, boolean reversed, Level resultsLevel)
// throws Exception;
- public Results getConnectingEntities(
- String connectionType, String entityType, Level level, int count) throws Exception;
+ public Results getSourceEntities(
+ String connectionType, String entityType, Level level, int count) throws Exception;
- public Results searchConnectedEntities( Query query ) throws Exception;
+ public Results searchTargetEntities(Query query) throws Exception;
public Set<String> getConnectionIndexes( String connectionType ) throws Exception;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5b1dfa1c/stack/core/src/test/java/org/apache/usergrid/persistence/CollectionIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/CollectionIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/CollectionIT.java
index e3149d6..eab04ce 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/CollectionIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/CollectionIT.java
@@ -1455,7 +1455,7 @@ public class CollectionIT extends AbstractCoreIT {
query.setConnectionType( "testconnection" );
query.setEntityType( "user" );
- r = em.searchConnectedEntities( foo, query );
+ r = em.searchTargetEntities(foo, query);
assertEquals( "connection must match", 1, r.size() );
assertEquals( entity.getUuid(), r.getId() );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5b1dfa1c/stack/core/src/test/java/org/apache/usergrid/persistence/CountingMutatorIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/CountingMutatorIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/CountingMutatorIT.java
index abb9417..63c7cb8 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/CountingMutatorIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/CountingMutatorIT.java
@@ -27,8 +27,6 @@ import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.commons.lang3.RandomStringUtils;
-
import org.apache.usergrid.AbstractCoreIT;
import org.apache.usergrid.persistence.hector.CountingMutator;
import org.apache.usergrid.persistence.Query.Level;
@@ -99,8 +97,8 @@ public class CountingMutatorIT extends AbstractCoreIT {
//now verify our connections were created properly
- PagingResultsIterator itr = new PagingResultsIterator(em.getConnectingEntities(
- returned, "following", "user", Level.ALL_PROPERTIES, 1000 ));
+ PagingResultsIterator itr = new PagingResultsIterator(em.getSourceEntities(
+ returned, "following", "user", Level.ALL_PROPERTIES, 1000));
int count = 0;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5b1dfa1c/stack/core/src/test/java/org/apache/usergrid/persistence/EntityConnectionsIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/EntityConnectionsIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/EntityConnectionsIT.java
index e1b2f5e..47b61e9 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/EntityConnectionsIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/EntityConnectionsIT.java
@@ -72,7 +72,7 @@ public class EntityConnectionsIT extends AbstractCoreIT {
app.refreshIndex();
- Results r = em.getConnectedEntities( firstUserEntity, "likes", null, Level.IDS );
+ Results r = em.getTargetEntities(firstUserEntity, "likes", null, Level.IDS);
List<ConnectionRef> connections = r.getConnections();
@@ -186,7 +186,7 @@ public class EntityConnectionsIT extends AbstractCoreIT {
EntityManager em = setup.getEmf().getEntityManager( applicationId );
Entity en = em.get( new SimpleEntityRef( entityType, entityId));
- Results results = em.getConnectedEntities( en, connectionType, null, Level.REFS );
+ Results results = em.getTargetEntities(en, connectionType, null, Level.REFS);
LOG.info( "----------------------------------------------------" );
assertEquals( "Expected " + expectedCount + " connections",
@@ -264,7 +264,7 @@ public class EntityConnectionsIT extends AbstractCoreIT {
app.refreshIndex();
- Results r = em.getConnectedEntities( firstUserEntity, "likes", "restaurant", Level.IDS );
+ Results r = em.getTargetEntities(firstUserEntity, "likes", "restaurant", Level.IDS);
List<ConnectionRef> connections = r.getConnections();
@@ -278,7 +278,7 @@ public class EntityConnectionsIT extends AbstractCoreIT {
assertFalse( em.isConnectionMember( firstUserEntity, "likes", arrogantbutcher ) );
// check we don't get the restaurant from the second user
- r = em.getConnectedEntities( secondUserEntity, "likes", "restaurant", Level.IDS );
+ r = em.getTargetEntities(secondUserEntity, "likes", "restaurant", Level.IDS);
connections = r.getConnections();
@@ -320,19 +320,19 @@ public class EntityConnectionsIT extends AbstractCoreIT {
// // search for "likes" edges from fred
// assertEquals( 1,
-// em.getConnectedEntities( fredEntity, "likes", null, Level.IDS ).size());
+// em.getTargetEntities( fredEntity, "likes", null, Level.IDS ).size());
//
// // search for any type of edges from fred
// assertEquals( 1,
-// em.getConnectedEntities( fredEntity, null, null, Level.IDS ).size());
+// em.getTargetEntities( fredEntity, null, null, Level.IDS ).size());
// search for "likes" edges to wilman from any type of object
- Results res = em.getConnectingEntities( wilmaEntity, "likes", null, Level.ALL_PROPERTIES);
+ Results res = em.getSourceEntities(wilmaEntity, "likes", null, Level.ALL_PROPERTIES);
assertEquals( 1, res.size() );
assertEquals( "user", res.getEntity().getType() ); // fred is a user
// search for "likes" edges to wilman from user type object
- res = em.getConnectingEntities( wilmaEntity, "likes", "user", Level.ALL_PROPERTIES);
+ res = em.getSourceEntities(wilmaEntity, "likes", "user", Level.ALL_PROPERTIES);
assertEquals( 1, res.size() );
assertEquals( "user", res.getEntity().getType() );
}
@@ -372,7 +372,7 @@ public class EntityConnectionsIT extends AbstractCoreIT {
app.refreshIndex();
- Results r = em.getConnectedEntities( firstUserEntity, "likes", null, Level.ALL_PROPERTIES ) ;
+ Results r = em.getTargetEntities(firstUserEntity, "likes", null, Level.ALL_PROPERTIES) ;
PagingResultsIterator itr = new PagingResultsIterator( r );
@@ -418,19 +418,19 @@ public class EntityConnectionsIT extends AbstractCoreIT {
//
//// // search for "likes" edges from fred
//// assertEquals( 1,
-//// em.getConnectedEntities( fredEntity, "likes", null, Level.IDS ).size());
+//// em.getTargetEntities( fredEntity, "likes", null, Level.IDS ).size());
////
//// // search for any type of edges from fred
//// assertEquals( 1,
-//// em.getConnectedEntities( fredEntity, null, null, Level.IDS ).size());
+//// em.getTargetEntities( fredEntity, null, null, Level.IDS ).size());
//
// // search for "likes" edges to wilman from any type of object
-// Results res = em.getConnectingEntities( wilmaEntity, "likes", null, Level.ALL_PROPERTIES);
+// Results res = em.getSourceEntities( wilmaEntity, "likes", null, Level.ALL_PROPERTIES);
// assertEquals( 1, res.size() );
// assertEquals( "user", res.getEntity().getType() ); // fred is a user
//
// // search for "likes" edges to wilman from user type object
-// res = em.getConnectingEntities( wilmaEntity, "likes", "user", Level.ALL_PROPERTIES);
+// res = em.getSourceEntities( wilmaEntity, "likes", "user", Level.ALL_PROPERTIES);
// assertEquals( 1, res.size() );
// assertEquals( "user", res.getEntity().getType() );
// }
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5b1dfa1c/stack/core/src/test/java/org/apache/usergrid/persistence/GeoIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/GeoIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/GeoIT.java
index 585735a..c837ab7 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/GeoIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/GeoIT.java
@@ -199,18 +199,18 @@ public class GeoIT extends AbstractCoreIT {
app.refreshIndex();
//4. Test that the user is within 2000m of the entity
- Results emSearchResults = em.searchConnectedEntities(user,
+ Results emSearchResults = em.searchTargetEntities(user,
Query.fromQL("location within 5000 of "
+ ((LinkedHashMap<String, Object>) userProperties.get("location")).get("latitude")
+ ", " + ((LinkedHashMap<String, Object>)
- userProperties.get("location")).get("longitude")).setConnectionType("likes"));
+ userProperties.get("location")).get("longitude")).setConnectionType("likes"));
assertEquals(1, emSearchResults.size());
//5. Test that the user is NOT within 1000m of the entity
- emSearchResults = em.searchConnectedEntities(user,
+ emSearchResults = em.searchTargetEntities(user,
Query.fromQL("location within 1000 of "
+ ((LinkedHashMap<String, Object>) userProperties.get("location")).get("latitude")
+ ", " + ((LinkedHashMap<String, Object>)
- userProperties.get("location")).get("longitude")).setConnectionType("likes"));
+ userProperties.get("location")).get("longitude")).setConnectionType("likes"));
assertEquals(0, emSearchResults.size());
//cleanup
em.delete(user);
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5b1dfa1c/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java b/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
index df2a762..181b5de 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
@@ -35,13 +35,9 @@ import org.slf4j.LoggerFactory;
import org.apache.usergrid.AbstractCoreIT;
import org.apache.usergrid.cassandra.SpringResource;
-import org.apache.usergrid.corepersistence.util.CpNamingUtils;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
-import org.apache.usergrid.persistence.index.EntityIndex;
import org.apache.usergrid.persistence.index.EntityIndexFactory;
-import org.apache.usergrid.persistence.index.impl.EsEntityIndexImpl;
-import org.apache.usergrid.persistence.Query;
import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.entity.SimpleId;
@@ -410,8 +406,8 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT {
assertEquals( 2000, e.getProperty("key2"));
- Results catResults = em.searchConnectedEntities(e,
- Query.fromQL("select *").setConnectionType( "herds" ));
+ Results catResults = em.searchTargetEntities(e,
+ Query.fromQL("select *").setConnectionType("herds"));
assertEquals( expectedConnections, catResults.size() );
if ( count % 100 == 0 ) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5b1dfa1c/stack/core/src/test/java/org/apache/usergrid/persistence/query/ConnectionHelper.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/query/ConnectionHelper.java b/stack/core/src/test/java/org/apache/usergrid/persistence/query/ConnectionHelper.java
index e99ae9c..e5c84f8 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/query/ConnectionHelper.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/query/ConnectionHelper.java
@@ -80,6 +80,6 @@ public class ConnectionHelper extends CollectionIoHelper {
query.setConnectionType( CONNECTION );
query.setEntityType( "test" );
- return app.getEntityManager().searchConnectedEntities( rootEntity, query );
+ return app.getEntityManager().searchTargetEntities(rootEntity, query);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5b1dfa1c/stack/core/src/test/java/org/apache/usergrid/persistence/query/IteratingQueryIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/query/IteratingQueryIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/query/IteratingQueryIT.java
index 17db7f5..c4a16b6 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/query/IteratingQueryIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/query/IteratingQueryIT.java
@@ -275,7 +275,7 @@ public class IteratingQueryIT {
query.setConnectionType( CONNECTION );
// don't set it on purpose
query.setEntityType( null );
- return app.getEntityManager().searchConnectedEntities( rootEntity, query );
+ return app.getEntityManager().searchTargetEntities(rootEntity, query);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5b1dfa1c/stack/rest/src/test/java/org/apache/usergrid/rest/management/ImportResourceIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/management/ImportResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/management/ImportResourceIT.java
index 70b9b69..e33d488 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/management/ImportResourceIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/management/ImportResourceIT.java
@@ -360,7 +360,7 @@ public class ImportResourceIT extends AbstractRestIT {
//
// int conCount = 0;
// for ( org.apache.usergrid.persistence.Entity e : importedThings ) {
-// Results r = emApp2.getConnectedEntities( e, "related", null, Query.Level.IDS);
+// Results r = emApp2.getTargetEntities( e, "related", null, Query.Level.IDS);
// List<ConnectionRef> connections = r.getConnections();
// conCount += connections.size();
// }
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5b1dfa1c/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
index 7909439..d9068bd 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
@@ -1730,9 +1730,9 @@ public class ManagementServiceImpl implements ManagementService {
final EntityManager em = emf.getEntityManager( smf.getManagementAppId() );
- Results r = em.getConnectingEntities(
- new SimpleEntityRef(CpNamingUtils.APPLICATION_INFO, applicationInfoId),
- "owns", Group.ENTITY_TYPE, Level.ALL_PROPERTIES );
+ Results r = em.getSourceEntities(
+ new SimpleEntityRef(CpNamingUtils.APPLICATION_INFO, applicationInfoId),
+ "owns", Group.ENTITY_TYPE, Level.ALL_PROPERTIES);
Entity entity = r.getEntity();
if ( entity != null ) {
@@ -1753,9 +1753,9 @@ public class ManagementServiceImpl implements ManagementService {
final EntityManager em = emf.getEntityManager( smf.getManagementAppId() );
// query for application_info entities
- final Results results = em.getConnectedEntities(
- new SimpleEntityRef(Group.ENTITY_TYPE, organizationGroupId),
- "owns", CpNamingUtils.APPLICATION_INFO, Level.ALL_PROPERTIES );
+ final Results results = em.getTargetEntities(
+ new SimpleEntityRef(Group.ENTITY_TYPE, organizationGroupId),
+ "owns", CpNamingUtils.APPLICATION_INFO, Level.ALL_PROPERTIES);
final PagingResultsIterator itr = new PagingResultsIterator( results );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5b1dfa1c/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
index ec3b86c..31e8544 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
@@ -508,9 +508,9 @@ public class ExportServiceImpl implements ExportService {
jg.writeFieldName( connectionType );
jg.writeStartArray();
- Results results = em.getConnectedEntities(
+ Results results = em.getTargetEntities(
new SimpleEntityRef(entity.getType(), entity.getUuid()),
- connectionType, null, Level.IDS );
+ connectionType, null, Level.IDS);
List<ConnectionRef> connections = results.getConnections();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5b1dfa1c/stack/services/src/main/java/org/apache/usergrid/management/importer/FileImportJob.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/importer/FileImportJob.java b/stack/services/src/main/java/org/apache/usergrid/management/importer/FileImportJob.java
index c656e96..f734b3f 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/importer/FileImportJob.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/importer/FileImportJob.java
@@ -19,7 +19,6 @@ package org.apache.usergrid.management.importer;
import org.apache.usergrid.batch.JobExecution;
import org.apache.usergrid.batch.job.OnlyOnceJob;
-import org.apache.usergrid.corepersistence.util.CpNamingUtils;
import org.apache.usergrid.persistence.Entity;
import org.apache.usergrid.persistence.EntityManager;
import org.apache.usergrid.persistence.EntityManagerFactory;
@@ -112,8 +111,8 @@ public class FileImportJob extends OnlyOnceJob {
rootEm.update(fileImport);
// If one file Job fails, mark the main import Job also as failed
- Results ImportJobResults = rootEm.getConnectingEntities(
- fileImport, "includes", null, Level.ALL_PROPERTIES);
+ Results ImportJobResults = rootEm.getSourceEntities(
+ fileImport, "includes", null, Level.ALL_PROPERTIES);
List<Entity> importEntity = ImportJobResults.getEntities();
UUID importId = importEntity.get(0).getUuid();
Import importUG = rootEm.get(importId, Import.class);
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5b1dfa1c/stack/services/src/main/java/org/apache/usergrid/management/importer/ImportServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/importer/ImportServiceImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/importer/ImportServiceImpl.java
index 18879c0..e446144 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/importer/ImportServiceImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/importer/ImportServiceImpl.java
@@ -40,7 +40,6 @@ import org.slf4j.LoggerFactory;
import rx.Observable;
import rx.Subscriber;
import rx.functions.Action1;
-import rx.functions.Func1;
import rx.functions.Func2;
import rx.schedulers.Schedulers;
@@ -213,7 +212,7 @@ public class ImportServiceImpl implements ImportService {
//set our entity type
query.setEntityType( Schema.getDefaultSchema().getEntityType( FileImport.class ) );
- return rootEm.searchConnectedEntities( importEntity, query );
+ return rootEm.searchTargetEntities(importEntity, query);
}
catch ( Exception e ) {
throw new RuntimeException( "Unable to get import entity", e );
@@ -272,7 +271,7 @@ public class ImportServiceImpl implements ImportService {
//set our entity type
query.setEntityType( Schema.getDefaultSchema().getEntityType( FailedImportEntity.class ) );
- return rootEm.searchConnectedEntities( importEntity, query );
+ return rootEm.searchTargetEntities(importEntity, query);
}
catch ( Exception e ) {
throw new RuntimeException( "Unable to get import entity", e );
@@ -382,11 +381,11 @@ public class ImportServiceImpl implements ImportService {
query.setLimit(MAX_FILE_IMPORTS);
// TODO, this won't work with more than 100 files
- Results entities = rootEM.searchConnectedEntities( importRoot, query );
+ Results entities = rootEM.searchTargetEntities(importRoot, query);
return entities.size();
// see ImportConnectsTest()
-// Results entities = rootEM.getConnectedEntities(
+// Results entities = rootEM.getTargetEntities(
// importRoot, "includes", null, Level.ALL_PROPERTIES );
// PagingResultsIterator itr = new PagingResultsIterator( entities );
// int count = 0;
@@ -767,8 +766,8 @@ public class ImportServiceImpl implements ImportService {
private Import getImportEntity( final EntityManager rootEm, final FileImport fileImport ) {
try {
Results importJobResults =
- rootEm.getConnectingEntities( fileImport, IMPORT_FILE_INCLUDES_CONNECTION,
- null, Level.ALL_PROPERTIES );
+ rootEm.getSourceEntities(fileImport, IMPORT_FILE_INCLUDES_CONNECTION,
+ null, Level.ALL_PROPERTIES);
List<Entity> importEntities = importJobResults.getEntities();
final Import importEntity = ( Import ) importEntities.get( 0 ).toTypedEntity();
@@ -802,7 +801,7 @@ public class ImportServiceImpl implements ImportService {
query.setConnectionType( IMPORT_FILE_INCLUDES_CONNECTION );
query.setLimit( MAX_FILE_IMPORTS );
- Results entities = rootEM.searchConnectedEntities( importEntity, query );
+ Results entities = rootEM.searchTargetEntities(importEntity, query);
PagingResultsIterator itr = new PagingResultsIterator( entities );
if ( !itr.hasNext() ) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5b1dfa1c/stack/services/src/main/java/org/apache/usergrid/services/AbstractConnectionsService.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/AbstractConnectionsService.java b/stack/services/src/main/java/org/apache/usergrid/services/AbstractConnectionsService.java
index 467b09e..c4142a0 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/AbstractConnectionsService.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/AbstractConnectionsService.java
@@ -172,14 +172,14 @@ public class AbstractConnectionsService extends AbstractService {
Results r = null;
if ( connecting() ) {
- r = em.getConnectingEntities(
- new SimpleEntityRef( context.getOwner().getType(), context.getOwner().getUuid()),
- context.getCollectionName(), null, Level.ALL_PROPERTIES );
+ r = em.getSourceEntities(
+ new SimpleEntityRef(context.getOwner().getType(), context.getOwner().getUuid()),
+ context.getCollectionName(), null, Level.ALL_PROPERTIES);
}
else {
- r = em.getConnectedEntities(
- new SimpleEntityRef( context.getOwner().getType(), context.getOwner().getUuid()),
- context.getCollectionName(), null, Level.ALL_PROPERTIES );
+ r = em.getTargetEntities(
+ new SimpleEntityRef(context.getOwner().getType(), context.getOwner().getUuid()),
+ context.getCollectionName(), null, Level.ALL_PROPERTIES);
}
importEntities( context, r );
@@ -302,17 +302,17 @@ public class AbstractConnectionsService extends AbstractService {
return null;
}
else {
-// r = em.getConnectingEntities( context.getOwner().getUuid(), query.getConnectionType(),
+// r = em.getSourceEntities( context.getOwner().getUuid(), query.getConnectionType(),
// query.getEntityType(), level );
// usergrid-2389: User defined limit in the query is ignored. Fixed it by adding
// the limit to the method parameter downstream.
- r = em.getConnectingEntities(
- new SimpleEntityRef( context.getOwner().getType(), context.getOwner().getUuid()),
- query.getConnectionType(),query.getEntityType(), level , query.getLimit());
+ r = em.getSourceEntities(
+ new SimpleEntityRef(context.getOwner().getType(), context.getOwner().getUuid()),
+ query.getConnectionType(), query.getEntityType(), level, query.getLimit());
}
}
else {
- r = em.searchConnectedEntities( context.getOwner(), query );
+ r = em.searchTargetEntities(context.getOwner(), query);
}
importEntities( context, r );
@@ -419,7 +419,7 @@ public class AbstractConnectionsService extends AbstractService {
}
- Results r = em.searchConnectedEntities( context.getOwner(), query );
+ Results r = em.searchTargetEntities(context.getOwner(), query);
if ( r.isEmpty() ) {
throw new ServiceResourceNotFoundException( context );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5b1dfa1c/stack/services/src/main/java/org/apache/usergrid/services/groups/users/activities/ActivitiesService.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/groups/users/activities/ActivitiesService.java b/stack/services/src/main/java/org/apache/usergrid/services/groups/users/activities/ActivitiesService.java
index ebf3676..014dcff 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/groups/users/activities/ActivitiesService.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/groups/users/activities/ActivitiesService.java
@@ -74,8 +74,8 @@ public class ActivitiesService extends GenericCollectionService {
return;
}
- Results r2 = em.getConnectingEntities( new SimpleEntityRef( user.getType(), user.getUuid()),
- "following", User.ENTITY_TYPE, Level.IDS );
+ Results r2 = em.getSourceEntities(new SimpleEntityRef(user.getType(), user.getUuid()),
+ "following", User.ENTITY_TYPE, Level.IDS);
if ( ( r2 == null ) || ( r2.isEmpty() ) ) {
return;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5b1dfa1c/stack/services/src/main/java/org/apache/usergrid/services/users/activities/ActivitiesService.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/users/activities/ActivitiesService.java b/stack/services/src/main/java/org/apache/usergrid/services/users/activities/ActivitiesService.java
index e7f3268..c297864 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/users/activities/ActivitiesService.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/users/activities/ActivitiesService.java
@@ -133,8 +133,8 @@ public class ActivitiesService extends GenericCollectionService {
em.addToCollection( user, "feed", activity );
//publish to all connections
- Results results = em.getConnectingEntities(
- new SimpleEntityRef( user.getType(), user.getUuid()),
+ Results results = em.getSourceEntities(
+ new SimpleEntityRef(user.getType(), user.getUuid()),
"following", User.ENTITY_TYPE, Level.REFS);
if( results != null ){
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5b1dfa1c/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportCollectionIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportCollectionIT.java b/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportCollectionIT.java
index 7350dcd..4c78a1e 100644
--- a/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportCollectionIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportCollectionIT.java
@@ -203,7 +203,7 @@ public class ImportCollectionIT {
int conCount = 0;
for ( Entity e : importedThings ) {
- Results r = emApp2.getConnectedEntities( e, "related", null, Level.IDS);
+ Results r = emApp2.getTargetEntities(e, "related", null, Level.IDS);
List<ConnectionRef> connections = r.getConnections();
conCount += connections.size();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5b1dfa1c/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportConnectionsTest.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportConnectionsTest.java b/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportConnectionsTest.java
index 2c9db95..8ca3caa 100644
--- a/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportConnectionsTest.java
+++ b/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportConnectionsTest.java
@@ -31,7 +31,6 @@ import org.apache.usergrid.NewOrgAppAdminRule;
import org.apache.usergrid.ServiceITSetup;
import org.apache.usergrid.ServiceITSetupImpl;
import org.apache.usergrid.cassandra.ClearShiroSubject;
-import org.apache.usergrid.corepersistence.util.CpNamingUtils;
import org.apache.usergrid.persistence.EntityManager;
import org.apache.usergrid.persistence.PagingResultsIterator;
import org.apache.usergrid.persistence.Results;
@@ -59,7 +58,7 @@ public class ImportConnectionsTest {
@Test
- @Ignore("Because getConnectedEntities() is broken")
+ @Ignore("Because getTargetEntities() is broken")
public void testCreateAndCountConnectionsViaGet() throws Exception {
doTestCreateAndCountConnections(new ConnectionCounter() {
@@ -135,8 +134,8 @@ public class ImportConnectionsTest {
EntityManager emMgmtApp = setup.getEmf()
.getEntityManager(setup.getEmf().getManagementAppId() );
- Results entities = emMgmtApp.getConnectedEntities(
- importRoot, "includes", null, Query.Level.ALL_PROPERTIES );
+ Results entities = emMgmtApp.getTargetEntities(
+ importRoot, "includes", null, Query.Level.ALL_PROPERTIES);
PagingResultsIterator itr = new PagingResultsIterator( entities );
int count = 0;
@@ -164,7 +163,7 @@ public class ImportConnectionsTest {
query.setConnectionType("includes");
query.setLimit(10000);
- Results entities = emMgmtApp.searchConnectedEntities( importRoot, query );
+ Results entities = emMgmtApp.searchTargetEntities(importRoot, query);
return entities.size();
// PagingResultsIterator itr = new PagingResultsIterator( entities );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5b1dfa1c/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportServiceIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportServiceIT.java b/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportServiceIT.java
index fd199e6..e8d6081 100644
--- a/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportServiceIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportServiceIT.java
@@ -28,7 +28,6 @@ import org.apache.usergrid.ServiceITSetup;
import org.apache.usergrid.ServiceITSetupImpl;
import org.apache.usergrid.batch.JobExecution;
import org.apache.usergrid.batch.service.JobSchedulerService;
-import org.apache.usergrid.cassandra.CassandraResource;
import org.apache.usergrid.cassandra.ClearShiroSubject;
import org.apache.usergrid.management.OrganizationInfo;
import org.apache.usergrid.management.UserInfo;
@@ -39,7 +38,6 @@ import org.apache.usergrid.persistence.*;
import org.apache.usergrid.persistence.entities.Import;
import org.apache.usergrid.persistence.entities.JobData;
import org.apache.usergrid.persistence.exceptions.EntityNotFoundException;
-import org.apache.usergrid.persistence.index.impl.ElasticSearchResource;
import org.apache.usergrid.persistence.Query.Level;
import org.apache.usergrid.persistence.index.utils.UUIDUtils;
import org.apache.usergrid.services.notifications.QueueListener;
@@ -248,7 +246,7 @@ public class ImportServiceIT {
Results r;
List<ConnectionRef> connections;
for (int i = 0; i < 2; i++) {
- r = em.getConnectedEntities(entities.get(i), "related", null, Level.IDS);
+ r = em.getTargetEntities(entities.get(i), "related", null, Level.IDS);
connections = r.getConnections();
assertNotNull(connections);
}
@@ -359,7 +357,7 @@ public class ImportServiceIT {
Results r;
List<ConnectionRef> connections;
for (int i = 0; i < 2; i++) {
- r = em.getConnectedEntities(entities.get(i), "related", null, Level.IDS);
+ r = em.getTargetEntities(entities.get(i), "related", null, Level.IDS);
connections = r.getConnections();
assertNotNull(connections);
}
[16/22] incubator-usergrid git commit: Merge branch 'pr/249' into
two-dot-o-dev
Posted by gr...@apache.org.
Merge branch 'pr/249' into two-dot-o-dev
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/b637535d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/b637535d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/b637535d
Branch: refs/heads/USERGRID-608
Commit: b637535ddeda1a9e74f07619864eace2ba0aca77
Parents: 4116ce5 7e917f4
Author: Shawn Feldman <sf...@apache.org>
Authored: Fri May 15 13:50:46 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Fri May 15 13:50:46 2015 -0600
----------------------------------------------------------------------
portal/config.js | 4 +++-
portal/js/global/ug-service.js | 4 ++--
2 files changed, 5 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
[02/22] incubator-usergrid git commit: Update of Apis and tests
Posted by gr...@apache.org.
Update of Apis and tests
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/dbfff997
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/dbfff997
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/dbfff997
Branch: refs/heads/USERGRID-608
Commit: dbfff9970f844f2264113fc05ee290aa3ca0ae50
Parents: 30e0e45
Author: Todd Nine <tn...@apigee.com>
Authored: Mon May 11 09:27:28 2015 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Tue May 12 10:29:41 2015 -0700
----------------------------------------------------------------------
.../corepersistence/CpEntityManagerFactory.java | 14 ------
.../corepersistence/index/ReIndexAction.java | 4 ++
.../corepersistence/index/ReIndexService.java | 11 ++--
.../index/ReIndexServiceImpl.java | 5 +-
.../persistence/EntityManagerFactory.java | 2 -
.../PerformanceEntityRebuildIndexTest.java | 53 ++++++--------------
6 files changed, 27 insertions(+), 62 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/dbfff997/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index e796545..13b6433 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -687,20 +687,6 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
}
}
- @Override
- public ReIndexService.IndexResponse rebuildCollectionIndex( Optional<UUID> appId, Optional<String> collection ) {
- throw new UnsupportedOperationException( "Implement me" );
-//
-// EntityManager em = getEntityManager( appId );
-//
-// //explicitly invoke create index, we don't know if it exists or not in ES during a rebuild.
-// Application app = em.getApplication();
-//
-// em.reindexCollection(po, collectionName, reverse);
-//
-// logger.info("\n\nRebuilt index for application {} id {} collection {}\n",
-// new Object[]{app.getName(), appId, collectionName});
- }
@Override
public void addIndex(final String indexSuffix,final int shards,final int replicas, final String writeConsistency){
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/dbfff997/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexAction.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexAction.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexAction.java
index 086b2aa..b878246 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexAction.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexAction.java
@@ -29,5 +29,9 @@ import org.apache.usergrid.persistence.collection.serialization.impl.migration.E
@FunctionalInterface
public interface ReIndexAction {
+ /**
+ * Index this entity with the specified scope
+ * @param entityIdScope
+ */
void index( final EntityIdScope entityIdScope );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/dbfff997/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java
index 91409fe..e594ad3 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java
@@ -44,12 +44,15 @@ public interface ReIndexService {
/**
* Perform an index rebuild
- * @param appId
- * @param collection
+ *
+ * @param appId The applicationId to re-index, or all applications if absent
+ * @param collection The collection name to re-index. Otherwise all collections in an app will be used.
+ * @param cursor An optional cursor to resume processing
+ * @param startTimestamp The time to start indexing from. All edges >= this time will be indexed.
* @return
*/
- IndexResponse rebuildIndex( final Optional<UUID> appId, final Optional<String> collection, final Optional<String> collectionName, final Optional<String> cursor,
- final Optional<Long> startTimestamp );
+ IndexResponse rebuildIndex( final Optional<UUID> appId, final Optional<String> collection, final Optional<String> cursor,
+ final Optional<Long> startTimestamp);
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/dbfff997/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
index be5bcab..bd1bff9 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
@@ -84,8 +84,7 @@ public class ReIndexServiceImpl implements ReIndexService {
@Override
- public IndexResponse rebuildIndex( final Optional<UUID> appId, final Optional<String> collection,
- final Optional<String> collectionName, final Optional<String> cursor,
+ public IndexResponse rebuildIndex( final Optional<UUID> appId, final Optional<String> collection, final Optional<String> cursor,
final Optional<Long> startTimestamp ) {
//load our last emitted Scope if a cursor is present
@@ -100,7 +99,7 @@ public class ReIndexServiceImpl implements ReIndexService {
//create an observable that loads each entity and indexes it, start it running with publish
final ConnectableObservable<EdgeScope> runningReIndex =
- allEntityIdsObservable.getEdgesToEntities( applicationScopes, collectionName, startTimestamp )
+ allEntityIdsObservable.getEdgesToEntities( applicationScopes, collection, startTimestamp )
//for each edge, create our scope and index on it
.doOnNext( edge -> indexService.index( new EntityIdScope( edge.getApplicationScope(), edge.getEdge().getTargetNode() ) ) ).publish();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/dbfff997/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
index b3f4b62..cd7e515 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
@@ -174,8 +174,6 @@ public interface EntityManagerFactory {
/** For testing purposes */
public void flushEntityManagerCaches();
- ReIndexService.IndexResponse rebuildCollectionIndex( Optional<UUID> appId, Optional<String> collection );
-
/**
* Add a new index to the application for scale
* @param suffix unique indentifier for additional index
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/dbfff997/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java b/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
index df2a762..a17c925 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
@@ -26,6 +26,8 @@ import java.util.concurrent.TimeUnit;
import com.google.common.base.Optional;
import org.apache.commons.lang.RandomStringUtils;
+
+import org.apache.usergrid.corepersistence.index.ReIndexService;
import org.apache.usergrid.persistence.index.ApplicationEntityIndex;
import org.junit.After;
import org.junit.Before;
@@ -63,9 +65,8 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT {
private static final MetricRegistry registry = new MetricRegistry();
private Slf4jReporter reporter;
- private static final long RUNTIME_MS = TimeUnit.SECONDS.toMillis( 10 );
+ private static final int ENTITIES_TO_INDEX = 2000;
- private static final long WRITE_DELAY_MS = 10;
@@ -99,6 +100,8 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT {
final EntityManager em = setup.getEmf().getEntityManager( appId );
+ final ReIndexService reIndexService = setup.getInjector().getInstance( ReIndexService.class );
+
// ----------------- create a bunch of entities
Map<String, Object> entityMap = new HashMap<String, Object>() {{
@@ -107,37 +110,18 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT {
put("key3", "Some value");
}};
-// Map<String, Object> cat1map = new HashMap<String, Object>() {{
-// put("name", "enzo");
-// put("color", "orange");
-// }};
-// Map<String, Object> cat2map = new HashMap<String, Object>() {{
-// put("name", "marquee");
-// put("color", "grey");
-// }};
-// Map<String, Object> cat3map = new HashMap<String, Object>() {{
-// put("name", "bertha");
-// put("color", "tabby");
-// }};
-//
-// Entity cat1 = em.create("cat", cat1map );
-// Entity cat2 = em.create("cat", cat2map );
-// Entity cat3 = em.create("cat", cat3map );
-
- final long stopTime = System.currentTimeMillis() + RUNTIME_MS;
List<EntityRef> entityRefs = new ArrayList<EntityRef>();
- int entityCount = 0;
int herderCount = 0;
int shepardCount = 0;
- while ( System.currentTimeMillis() < stopTime ) {
+ for (int i = 0; i < ENTITIES_TO_INDEX; i++) {
final Entity entity;
try {
- entityMap.put("key", entityCount );
+ entityMap.put("key", i );
- if ( entityCount % 2 == 0 ) {
+ if ( i % 2 == 0 ) {
entity = em.create("catherder", entityMap);
herderCount++;
} else {
@@ -156,15 +140,13 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT {
}
entityRefs.add(new SimpleEntityRef( entity.getType(), entity.getUuid() ) );
- if ( entityCount % 10 == 0 ) {
- logger.info("Created {} entities", entityCount );
+ if ( i % 10 == 0 ) {
+ logger.info("Created {} entities", i );
}
- entityCount++;
- try { Thread.sleep( WRITE_DELAY_MS ); } catch (InterruptedException ignored ) {}
}
- logger.info("Created {} entities", entityCount);
+ logger.info("Created {} entities", ENTITIES_TO_INDEX);
app.refreshIndex();
// ----------------- test that we can read them, should work fine
@@ -182,7 +164,7 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT {
logger.debug("Reading data, should fail this time ");
try {
- readData( em, "testTypes", entityCount, 0 );
+ readData( em, "testTypes", ENTITIES_TO_INDEX, 0 );
fail("should have failed to read data");
} catch (Exception expected) {}
@@ -214,10 +196,7 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT {
try {
-// // do it forwards
- setup.getEmf().rebuildCollectionIndex(Optional.of(em.getApplicationId()), Optional.<String>of("catherders"));
-//
-// // and backwards, just to make sure both cases are covered
+ reIndexService.rebuildIndex( Optional.of( em.getApplicationId()), Optional.<String>of("catherders"), Optional.absent(), Optional.absent() );
reporter.report();
registry.remove( meterName );
@@ -269,11 +248,9 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT {
Entity cat2 = em.create("cat", cat2map );
Entity cat3 = em.create("cat", cat3map );
- final long stopTime = System.currentTimeMillis() + RUNTIME_MS;
-
List<EntityRef> entityRefs = new ArrayList<EntityRef>();
int entityCount = 0;
- while ( System.currentTimeMillis() < stopTime ) {
+ for (int i = 0; i < ENTITIES_TO_INDEX; i++) {
final Entity entity;
@@ -295,8 +272,6 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT {
logger.info("Created {} entities", entityCount );
}
- entityCount++;
- try { Thread.sleep( WRITE_DELAY_MS ); } catch (InterruptedException ignored ) {}
}
logger.info("Created {} entities", entityCount);
[12/22] incubator-usergrid git commit: Change usergrid SDK logging to
be set via config.
Posted by gr...@apache.org.
Change usergrid SDK logging to be set via config.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/7e917f43
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/7e917f43
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/7e917f43
Branch: refs/heads/USERGRID-608
Commit: 7e917f43367e6958cf075dbd6a500cbe39138941
Parents: 5b1dfa1
Author: Michael Russo <mr...@apigee.com>
Authored: Thu May 14 18:22:01 2015 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Thu May 14 18:31:12 2015 -0700
----------------------------------------------------------------------
portal/config.js | 4 +++-
portal/js/global/ug-service.js | 4 ++--
2 files changed, 5 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7e917f43/portal/config.js
----------------------------------------------------------------------
diff --git a/portal/config.js b/portal/config.js
index b84c62e..82c5a0f 100644
--- a/portal/config.js
+++ b/portal/config.js
@@ -27,7 +27,9 @@ Usergrid.overrideUrl = 'http://localhost:8080/';
Usergrid.options = {
client: {
- requiresDeveloperKey: false
+ requiresDeveloperKey: false,
+ buildCurl: true,
+ logging: true
// apiKey:'123456'
},
showAutoRefresh: true,
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7e917f43/portal/js/global/ug-service.js
----------------------------------------------------------------------
diff --git a/portal/js/global/ug-service.js b/portal/js/global/ug-service.js
index 230d54c..7bc9d7c 100644
--- a/portal/js/global/ug-service.js
+++ b/portal/js/global/ug-service.js
@@ -181,8 +181,8 @@ AppServices.Services.factory('ug', function(configuration, $rootScope, utility,
},
client: function() {
var options = {
- buildCurl: true,
- logging: true
+ buildCurl: Usergrid.options.client.buildCurl || false,
+ logging: Usergrid.options.client.logging || false
};
if (Usergrid.options && Usergrid.options.client) {
options.keys = Usergrid.options.client;
[06/22] incubator-usergrid git commit: merge
Posted by gr...@apache.org.
merge
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/ed8ee9a9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/ed8ee9a9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/ed8ee9a9
Branch: refs/heads/USERGRID-608
Commit: ed8ee9a9a197b00497a1f7b74701368e2b48763b
Parents: 072e2d4 e3ba2b0
Author: Shawn Feldman <sf...@apache.org>
Authored: Wed May 13 10:39:21 2015 -0700
Committer: Shawn Feldman <sf...@apache.org>
Committed: Wed May 13 10:39:21 2015 -0700
----------------------------------------------------------------------
portal/index-template.html | 2 +-
portal/js/data/data-controller.js | 58 +-
portal/js/data/data.html | 36 +-
portal/js/global/ug-service.js | 30 +-
portal/js/login/login-controller.js | 15 +-
.../Usergrid.Notifications/BackgroundTask.cs | 57 +
.../Usergrid.Notifications/Client/PushClient.cs | 12 +-
.../Usergrid.Notifications/MainPage.xaml.cs | 12 +-
.../Usergrid.Notifications/MyBackgroundTask.cs | 18 +
.../Usergrid.Notifications/Package.appxmanifest | 8 +
.../Usergrid.Notifications.csproj | 1 +
.../Usergrid.Notifications.sln | 40 +
.../main/dist/init_instance/init_rest_server.sh | 6 +-
.../main/resources/usergrid-default.properties | 5 +-
stack/core/pom.xml | 58 +-
.../batch/service/JobSchedulerService.java | 28 +-
.../batch/service/SchedulerService.java | 2 +-
.../batch/service/SchedulerServiceImpl.java | 23 +-
.../corepersistence/ApplicationIdCache.java | 51 +
.../ApplicationIdCacheFactory.java | 43 +
.../corepersistence/ApplicationIdCacheFig.java | 43 +
.../corepersistence/ApplicationIdCacheImpl.java | 146 +
.../usergrid/corepersistence/CoreModule.java | 156 +-
.../corepersistence/CpEntityManager.java | 449 +--
.../corepersistence/CpEntityManagerFactory.java | 572 ++--
.../corepersistence/CpManagerCache.java | 65 +-
.../corepersistence/CpQueryProcessor.java | 94 -
.../corepersistence/CpRelationManager.java | 1980 +++---------
.../usergrid/corepersistence/CpSetup.java | 32 +-
.../usergrid/corepersistence/CpWalker.java | 85 +-
.../corepersistence/EntityManagerFig.java | 38 +
.../usergrid/corepersistence/GuiceFactory.java | 75 +-
.../usergrid/corepersistence/ManagerCache.java | 17 +-
.../corepersistence/OrgApplicationCache.java | 67 -
.../OrgApplicationCacheImpl.java | 181 --
.../asyncevents/AsyncEventService.java | 71 +
.../asyncevents/AsyncIndexProvider.java | 124 +
.../asyncevents/EventBuilder.java | 105 +
.../asyncevents/EventBuilderImpl.java | 154 +
.../asyncevents/InMemoryAsyncEventService.java | 116 +
.../asyncevents/SQSAsyncEventService.java | 349 +++
.../events/EntityDeletedHandler.java | 82 -
.../events/EntityVersionCreatedHandler.java | 89 -
.../events/EntityVersionDeletedHandler.java | 106 -
.../corepersistence/index/IndexEntityEvent.java | 75 +
.../index/IndexProcessorFig.java | 89 +
.../corepersistence/index/IndexService.java | 85 +
.../corepersistence/index/IndexServiceImpl.java | 229 ++
.../corepersistence/index/ReIndexAction.java | 33 +
.../corepersistence/index/ReIndexService.java | 89 +
.../index/ReIndexServiceImpl.java | 130 +
.../migration/AppInfoMigrationPlugin.java | 292 ++
.../migration/AppInfoVersions.java | 40 +
.../migration/CoreDataVersions.java | 45 +
.../migration/CoreMigration.java | 53 +
.../migration/CoreMigrationPlugin.java | 64 +
.../migration/EntityDataMigration.java | 147 -
.../migration/EntityTypeMappingMigration.java | 69 +-
.../migration/GraphShardVersionMigration.java | 152 -
.../migration/MigrationModuleVersionPlugin.java | 145 +
.../migration/MigrationSystemVersions.java | 43 +
.../corepersistence/migration/Versions.java | 14 -
.../corepersistence/pipeline/Pipeline.java | 121 +
.../pipeline/PipelineBuilderFactory.java | 39 +
.../pipeline/PipelineContext.java | 82 +
.../pipeline/PipelineDiagram.jpg | Bin 0 -> 183035 bytes
.../pipeline/PipelineModule.java | 58 +
.../pipeline/PipelineOperation.java | 39 +
.../usergrid/corepersistence/pipeline/README.md | 187 ++
.../cursor/AbstractCursorSerializer.java | 62 +
.../pipeline/cursor/CursorDiagram.jpg | Bin 0 -> 167390 bytes
.../pipeline/cursor/CursorParseException.java | 30 +
.../pipeline/cursor/CursorSerializer.java | 42 +
.../pipeline/cursor/CursorSerializerUtil.java | 43 +
.../pipeline/cursor/RequestCursor.java | 111 +
.../pipeline/cursor/ResponseCursor.java | 110 +
.../pipeline/read/AbstractFilter.java | 45 +
.../pipeline/read/AbstractPathFilter.java | 109 +
.../pipeline/read/Collector.java | 38 +
.../pipeline/read/CollectorFactory.java | 38 +
.../corepersistence/pipeline/read/EdgePath.java | 79 +
.../corepersistence/pipeline/read/Filter.java | 31 +
.../pipeline/read/FilterFactory.java | 143 +
.../pipeline/read/FilterResult.java | 56 +
.../pipeline/read/ReadDiagram.jpg | Bin 0 -> 818565 bytes
.../pipeline/read/ReadFilterFactoryImpl.java | 136 +
.../pipeline/read/ReadPipelineBuilder.java | 104 +
.../pipeline/read/ReadPipelineBuilderImpl.java | 296 ++
.../pipeline/read/ResultsPage.java | 67 +
.../read/collect/AbstractCollector.java | 46 +
.../pipeline/read/collect/CollectDiagram.jpg | Bin 0 -> 98007 bytes
.../pipeline/read/collect/EntityFilter.java | 68 +
.../read/collect/IdCursorSerializer.java | 41 +
.../read/collect/ResultsPageCollector.java | 80 +
.../AbstractElasticSearchFilter.java | 171 +
.../pipeline/read/elasticsearch/Candidate.java | 55 +
.../elasticsearch/CandidateEntityFilter.java | 234 ++
.../read/elasticsearch/CandidateIdFilter.java | 201 ++
.../ElasticSearchCollectionFilter.java | 77 +
.../ElasticSearchConnectionFilter.java | 73 +
.../ElasticsearchCursorSerializer.java | 42 +
.../read/elasticsearch/Elasticsearchdiagram.jpg | Bin 0 -> 316655 bytes
.../graph/AbstractReadGraphEdgeByIdFilter.java | 82 +
.../read/graph/AbstractReadGraphFilter.java | 147 +
.../read/graph/EdgeCursorSerializer.java | 42 +
.../pipeline/read/graph/EntityIdFilter.java | 54 +
.../pipeline/read/graph/EntityLoadFilter.java | 155 +
.../pipeline/read/graph/GraphDiagram.jpg | Bin 0 -> 347711 bytes
.../graph/ReadGraphCollectionByIdFilter.java | 49 +
.../read/graph/ReadGraphCollectionFilter.java | 53 +
.../graph/ReadGraphConnectionByIdFilter.java | 50 +
.../graph/ReadGraphConnectionByTypeFilter.java | 100 +
.../read/graph/ReadGraphConnectionFilter.java | 53 +
.../results/CollectionRefsVerifier.java | 44 -
.../CollectionResultsLoaderFactoryImpl.java | 60 -
.../results/ConnectionRefsVerifier.java | 61 -
.../ConnectionResultsLoaderFactoryImpl.java | 65 -
.../results/ElasticSearchQueryExecutor.java | 216 --
.../corepersistence/results/EntityVerifier.java | 127 -
.../results/FilteringLoader.java | 254 --
.../corepersistence/results/IdsVerifier.java | 46 -
.../results/ObservableQueryExecutor.java | 135 +
.../corepersistence/results/ResultsLoader.java | 43 -
.../results/ResultsLoaderFactory.java | 42 -
.../results/ResultsVerifier.java | 52 -
.../results/VersionVerifier.java | 85 -
.../rx/AllEntitiesInSystemObservable.java | 101 -
.../rx/ApplicationObservable.java | 128 -
.../rx/EdgesFromSourceObservable.java | 63 -
.../rx/EdgesToTargetObservable.java | 63 -
.../corepersistence/rx/TargetIdObservable.java | 66 -
.../rx/impl/AbstractGraphVisitorImpl.java | 65 +
.../rx/impl/AllApplicationsObservable.java | 43 +
.../rx/impl/AllApplicationsObservableImpl.java | 134 +
.../rx/impl/AllEntitiesInSystemImpl.java | 53 +
.../rx/impl/AllEntityIdsObservable.java | 52 +
.../rx/impl/AllEntityIdsObservableImpl.java | 92 +
.../rx/impl/AllNodesInGraphImpl.java | 54 +
.../corepersistence/rx/impl/EdgeScope.java | 51 +
.../corepersistence/util/CpEntityMapUtils.java | 19 +-
.../corepersistence/util/CpNamingUtils.java | 236 +-
.../util/SerializableMapper.java | 91 +
.../usergrid/exception/ConflictException.java | 36 +
.../main/java/org/apache/usergrid/mq/Query.java | 2 +-
.../org/apache/usergrid/mq/QueryProcessor.java | 210 +-
.../apache/usergrid/mq/cassandra/QueuesCF.java | 5 -
.../usergrid/persistence/ConnectionRef.java | 4 +-
.../usergrid/persistence/CounterQuery.java | 2 +-
.../usergrid/persistence/EntityManager.java | 49 +-
.../persistence/EntityManagerFactory.java | 134 +-
.../persistence/EntityPropertyComparator.java | 56 -
.../persistence/GuiceAdapterBeanFactory.java | 114 +
.../persistence/MultiQueryIterator.java | 4 +-
.../persistence/ObservableIterator.java | 1 +
.../persistence/PagingResultsIterator.java | 4 +-
.../apache/usergrid/persistence/PathQuery.java | 6 +-
.../usergrid/persistence/PersistenceModule.java | 66 +
.../org/apache/usergrid/persistence/Query.java | 890 ++++++
.../apache/usergrid/persistence/QueryUtils.java | 61 +-
.../usergrid/persistence/RelationManager.java | 17 +-
.../apache/usergrid/persistence/Results.java | 81 +-
.../org/apache/usergrid/persistence/Schema.java | 5 +-
.../persistence/cassandra/ApplicationCF.java | 37 -
.../cassandra/CassandraPersistenceUtils.java | 122 +-
.../persistence/cassandra/CassandraService.java | 304 +-
.../cassandra/ConnectionRefImpl.java | 24 +-
.../persistence/cassandra/CursorCache.java | 137 -
.../cassandra/EntityManagerFactoryImpl.java | 485 ---
.../cassandra/EntityManagerImpl.java | 2935 ------------------
.../cassandra/EntityValueSerializer.java | 53 -
.../persistence/cassandra/GeoIndexManager.java | 330 --
.../persistence/cassandra/IndexUpdate.java | 451 ---
.../persistence/cassandra/QueryProcessor.java | 60 -
.../cassandra/QueryProcessorImpl.java | 727 -----
.../cassandra/RelationManagerImpl.java | 2298 --------------
.../persistence/cassandra/SetupImpl.java | 170 -
.../cassandra/SimpleIndexBucketLocatorImpl.java | 122 -
.../cassandra/index/ConnectedIndexScanner.java | 280 --
.../cassandra/index/IndexBucketScanner.java | 238 --
.../index/IndexMultiBucketSetLoader.java | 139 -
.../cassandra/index/IndexScanner.java | 40 -
.../cassandra/index/NoOpIndexScanner.java | 95 -
.../DuplicateUniquePropertyExistsException.java | 2 +-
.../persistence/geo/CollectionGeoSearch.java | 68 -
.../persistence/geo/ConnectionGeoSearch.java | 67 -
.../persistence/geo/EntityLocationRef.java | 227 --
.../EntityLocationRefDistanceComparator.java | 78 -
.../persistence/geo/GeoIndexSearcher.java | 370 ---
.../persistence/geo/GeocellManager.java | 195 --
.../usergrid/persistence/geo/GeocellUtils.java | 543 ----
.../geo/comparator/DoubleTupleComparator.java | 39 -
.../persistence/geo/model/BoundingBox.java | 74 -
.../persistence/geo/model/CostFunction.java | 36 -
.../geo/model/DefaultCostFunction.java | 37 -
.../usergrid/persistence/geo/model/Point.java | 61 -
.../usergrid/persistence/geo/model/Tuple.java | 40 -
.../usergrid/persistence/query/ir/AllNode.java | 82 -
.../usergrid/persistence/query/ir/AndNode.java | 44 -
.../persistence/query/ir/BooleanNode.java | 65 -
.../query/ir/EmailIdentifierNode.java | 58 -
.../query/ir/NameIdentifierNode.java | 56 -
.../persistence/query/ir/NodeVisitor.java | 79 -
.../usergrid/persistence/query/ir/NotNode.java | 75 -
.../usergrid/persistence/query/ir/OrNode.java | 53 -
.../persistence/query/ir/OrderByNode.java | 105 -
.../persistence/query/ir/QueryNode.java | 41 -
.../persistence/query/ir/QuerySlice.java | 345 --
.../persistence/query/ir/SearchVisitor.java | 270 --
.../persistence/query/ir/SliceNode.java | 180 --
.../query/ir/UuidIdentifierNode.java | 60 -
.../persistence/query/ir/WithinNode.java | 109 -
.../query/ir/result/AbstractScanColumn.java | 83 -
.../result/CollectionResultsLoaderFactory.java | 41 -
.../ir/result/ConnectionIndexSliceParser.java | 87 -
.../query/ir/result/ConnectionRefLoader.java | 81 -
.../result/ConnectionResultsLoaderFactory.java | 50 -
.../ir/result/ConnectionTypesIterator.java | 190 --
.../query/ir/result/EmptyIterator.java | 63 -
.../query/ir/result/EntityRefLoader.java | 51 -
.../query/ir/result/EntityResultsLoader.java | 47 -
.../query/ir/result/GeoIterator.java | 351 ---
.../persistence/query/ir/result/IDLoader.java | 40 -
.../query/ir/result/IntersectionIterator.java | 170 -
.../query/ir/result/MergeIterator.java | 150 -
.../query/ir/result/MultiIterator.java | 55 -
.../query/ir/result/OrderByIterator.java | 250 --
.../query/ir/result/ResultIterator.java | 44 -
.../query/ir/result/ResultsLoader.java | 30 -
.../query/ir/result/ResultsLoaderFactory.java | 37 -
.../persistence/query/ir/result/ScanColumn.java | 32 -
.../query/ir/result/ScanColumnTransformer.java | 39 -
.../ir/result/SecondaryIndexSliceParser.java | 61 -
.../query/ir/result/SliceIterator.java | 242 --
.../query/ir/result/SliceParser.java | 32 -
.../query/ir/result/StaticIdIterator.java | 82 -
.../query/ir/result/SubtractionIterator.java | 113 -
.../query/ir/result/UUIDIndexSliceParser.java | 47 -
.../query/ir/result/UnionIterator.java | 264 --
.../main/resources/usergrid-core-context.xml | 32 +-
.../java/org/apache/usergrid/Application.java | 2 +-
.../org/apache/usergrid/CoreApplication.java | 40 +-
.../java/org/apache/usergrid/CoreITSetup.java | 10 +-
.../org/apache/usergrid/CoreITSetupImpl.java | 69 +-
.../org/apache/usergrid/TestEntityIndex.java | 27 +
.../usergrid/batch/job/SchedulerRuntime8IT.java | 8 +-
.../corepersistence/StaleIndexCleanupTest.java | 124 +-
.../corepersistence/TestIndexModule.java | 43 +
.../index/AsyncIndexServiceTest.java | 199 ++
.../index/InMemoryAsycIndexServiceTest.java | 65 +
.../corepersistence/index/IndexServiceTest.java | 328 ++
.../corepersistence/index/PublishRxTest.java | 95 +
.../index/SQSAsyncEventServiceTest.java | 84 +
.../migration/EntityDataMigrationIT.java | 262 --
.../migration/EntityTypeMappingMigrationIT.java | 183 +-
.../migration/GraphShardVersionMigrationIT.java | 226 --
.../MigrationModuleVersionPluginTest.java | 259 ++
.../migration/MigrationTestRule.java | 99 -
.../migration/TestProgressObserver.java | 71 -
.../pipeline/cursor/CursorTest.java | 109 +
.../rx/AllEntitiesInSystemObservableIT.java | 71 +-
.../rx/ApplicationObservableTestIT.java | 17 +-
.../rx/EdgesFromSourceObservableIT.java | 19 +-
.../rx/EdgesToTargetObservableIT.java | 59 +-
.../rx/TargetIdObservableTestIT.java | 7 +-
.../java/org/apache/usergrid/mq/MessagesIT.java | 6 +
.../usergrid/persistence/CollectionIT.java | 388 +--
.../apache/usergrid/persistence/CounterIT.java | 11 +-
.../usergrid/persistence/CountingMutatorIT.java | 6 +-
.../persistence/EntityConnectionsIT.java | 36 +-
.../usergrid/persistence/EntityManagerIT.java | 66 +-
.../EntityPropertyComparatorTest.java | 222 --
.../org/apache/usergrid/persistence/GeoIT.java | 170 +-
.../persistence/GeoQueryBooleanTest.java | 28 +-
.../apache/usergrid/persistence/IndexIT.java | 62 +-
.../usergrid/persistence/LargeEntityIT.java | 3 +-
.../usergrid/persistence/PathQueryIT.java | 33 +-
.../PerformanceEntityRebuildIndexTest.java | 61 +-
.../usergrid/persistence/PermissionsIT.java | 6 +-
.../apache/usergrid/persistence/QueryTest.java | 328 --
.../cassandra/EntityManagerFactoryImplIT.java | 126 +-
.../cassandra/QueryProcessorTest.java | 822 -----
.../SimpleIndexShardLocatorImplTest.java | 177 --
...EntityLocationRefDistanceComparatorTest.java | 104 -
.../persistence/query/CollectionIoHelper.java | 3 +-
.../persistence/query/ConnectionHelper.java | 4 +-
.../query/IntersectionTransitivePagingIT.java | 7 +-
.../query/IntersectionUnionPagingIT.java | 5 +-
.../usergrid/persistence/query/IoHelper.java | 2 +-
.../persistence/query/IteratingQueryIT.java | 165 +-
.../persistence/query/NotSubPropertyIT.java | 8 +-
.../query/OrderByDiscardPagesPagingIT.java | 164 -
.../query/ir/result/AbstractScanColumnTest.java | 101 -
.../query/ir/result/InOrderIterator.java | 132 -
.../ir/result/IntersectionIteratorTest.java | 308 --
.../query/ir/result/IteratorHelper.java | 34 -
.../ir/result/SubtractionIteratorTest.java | 182 --
.../query/ir/result/UnionIteratorTest.java | 468 ---
.../apache/usergrid/utils/EdgeTestUtils.java | 50 +
stack/core/src/test/resources/log4j.properties | 4 +
.../resources/usergrid-custom-test.properties | 7 +-
.../resources/usergrid-properties-context.xml | 40 -
.../usergrid-scheduler-test.properties | 2 +
.../test/resources/usergrid-test-context.xml | 1 +
stack/corepersistence/collection/pom.xml | 14 +-
.../persistence/collection/CollectionScope.java | 45 -
.../collection/EntityCollectionManager.java | 67 +-
.../EntityCollectionManagerFactory.java | 36 +-
.../collection/EntityCollectionManagerSync.java | 49 -
.../persistence/collection/EntitySet.java | 14 +-
.../persistence/collection/FieldSet.java | 2 +-
.../persistence/collection/MvccEntity.java | 5 +-
.../persistence/collection/MvccLogEntry.java | 1 +
.../cache/CachedEntityCollectionManager.java | 29 +-
.../collection/event/EntityDeleted.java | 45 -
.../collection/event/EntityVersionCreated.java | 38 -
.../collection/event/EntityVersionDeleted.java | 45 -
.../exception/CollectionRuntimeException.java | 30 +-
.../exception/EntityTooLargeException.java | 6 +-
.../exception/WriteCommitException.java | 11 +-
.../WriteOptimisticVerifyException.java | 11 +-
.../exception/WriteStartException.java | 11 +-
.../exception/WriteUniqueVerifyException.java | 6 +-
.../collection/guice/CollectionModule.java | 63 +-
.../guice/CollectionTaskExecutor.java | 4 +-
.../persistence/collection/guice/Write.java | 17 -
.../collection/guice/WriteUpdate.java | 17 -
.../collection/impl/CollectionScopeImpl.java | 102 -
.../EntityCollectionManagerFactoryImpl.java | 124 +-
.../impl/EntityCollectionManagerImpl.java | 485 ++-
.../impl/EntityCollectionManagerSyncImpl.java | 71 -
.../collection/impl/EntityDeletedTask.java | 147 -
.../impl/EntityVersionCleanupTask.java | 248 --
.../impl/EntityVersionCreatedTask.java | 122 -
.../impl/EntityVersionTaskFactory.java | 65 -
.../mvcc/MvccEntitySerializationStrategy.java | 102 -
.../mvcc/MvccLogEntrySerializationStrategy.java | 80 -
.../collection/mvcc/changelog/ChangeLog.java | 2 -
.../mvcc/entity/MvccValidationUtils.java | 16 -
.../mvcc/entity/impl/MvccEntityDeleteEvent.java | 8 +-
.../mvcc/entity/impl/MvccEntityEvent.java | 11 +-
.../mvcc/entity/impl/MvccEntityWriteEvent.java | 6 +-
.../mvcc/stage/CollectionIoEvent.java | 9 +-
.../mvcc/stage/EntityUpdateEvent.java | 34 -
.../mvcc/stage/delete/MarkCommit.java | 15 +-
.../collection/mvcc/stage/delete/MarkStart.java | 20 +-
.../mvcc/stage/delete/UniqueCleanup.java | 133 +
.../mvcc/stage/delete/VersionCompact.java | 125 +
.../mvcc/stage/write/RollbackAction.java | 14 +-
.../mvcc/stage/write/WriteCommit.java | 32 +-
.../mvcc/stage/write/WriteOptimisticVerify.java | 12 +-
.../collection/mvcc/stage/write/WriteStart.java | 22 +-
.../mvcc/stage/write/WriteUniqueVerify.java | 45 +-
.../collection/serialization/EntityRepair.java | 38 -
.../MvccEntitySerializationStrategy.java | 117 +
.../MvccLogEntrySerializationStrategy.java | 96 +
.../serialization/OptimisticUpdate.java | 23 -
.../serialization/SerializationFig.java | 30 +-
.../collection/serialization/UniqueValue.java | 1 -
.../UniqueValueSerializationStrategy.java | 65 +-
.../serialization/UniqueValueSet.java | 1 +
.../impl/CollectionDataVersions.java | 54 +
.../serialization/impl/EntityRepairImpl.java | 149 -
.../serialization/impl/EntitySetImpl.java | 10 +-
.../impl/EntityVersionSerializer.java | 11 +-
.../serialization/impl/FieldSerializer.java | 118 -
.../serialization/impl/LogEntryIterator.java | 114 -
.../impl/MinMaxLogEntryIterator.java | 121 +
.../serialization/impl/MutableFieldSet.java | 6 +-
.../MvccEntitySerializationStrategyImpl.java | 177 +-
...vccEntitySerializationStrategyProxyImpl.java | 163 +-
.../MvccEntitySerializationStrategyV1Impl.java | 9 +-
.../MvccEntitySerializationStrategyV2Impl.java | 8 +-
.../MvccEntitySerializationStrategyV3Impl.java | 551 ++++
.../MvccLogEntrySerializationProxyImpl.java | 163 +
.../MvccLogEntrySerializationStrategyImpl.java | 157 +-
...MvccLogEntrySerializationStrategyV1Impl.java | 118 +
...MvccLogEntrySerializationStrategyV2Impl.java | 106 +
.../serialization/impl/SerializationModule.java | 170 +-
.../serialization/impl/TypeField.java | 43 +
.../serialization/impl/UniqueFieldEntry.java | 60 +
.../impl/UniqueFieldEntrySerializer.java | 138 +
.../impl/UniqueFieldRowKeySerializer.java | 122 +
.../impl/UniqueTypeFieldRowKeySerializer.java | 60 +
.../serialization/impl/UniqueValueImpl.java | 5 +-
.../UniqueValueSerializationStrategyImpl.java | 322 +-
...iqueValueSerializationStrategyProxyImpl.java | 179 ++
.../UniqueValueSerializationStrategyV1Impl.java | 159 +
.../UniqueValueSerializationStrategyV2Impl.java | 139 +
.../impl/migration/CollectionMigration.java | 53 +
.../migration/CollectionMigrationPlugin.java | 68 +
.../impl/migration/EntityIdScope.java | 79 +
.../migration/MvccEntityDataMigrationImpl.java | 270 ++
.../serialization/impl/util/Inflector.java | 370 +++
.../impl/util/LegacyScopeUtils.java | 51 +
.../collection/util/EntityUtils.java | 71 -
.../collection/ApplicationContextImplTest.java | 33 +
.../collection/CollectionContextImplTest.java | 53 -
.../EntityCollectionManagerFactoryTest.java | 9 +-
.../collection/EntityCollectionManagerIT.java | 319 +-
.../EntityCollectionManagerStressTest.java | 14 +-
.../EntityCollectionManagerSyncIT.java | 193 --
.../collection/guice/TestCollectionModule.java | 22 +-
.../impl/EntityVersionCleanupTaskTest.java | 808 -----
.../impl/EntityVersionCreatedTaskTest.java | 244 --
.../mvcc/stage/AbstractEntityStageTest.java | 6 +-
.../mvcc/stage/AbstractIdStageTest.java | 4 +-
.../mvcc/stage/AbstractMvccEntityStageTest.java | 6 +-
.../mvcc/stage/TestEntityGenerator.java | 2 +-
.../mvcc/stage/delete/MarkCommitTest.java | 14 +-
.../mvcc/stage/delete/MarkStartTest.java | 6 +-
.../mvcc/stage/delete/UniqueCleanupTest.java | 712 +++++
.../mvcc/stage/delete/VersionCompactTest.java | 238 ++
.../mvcc/stage/write/FieldSerializerTest.java | 52 -
.../write/UniqueFieldRowKeySerializerTest.java | 52 +
...niqueValueSerializationStrategyImplTest.java | 185 --
.../mvcc/stage/write/WriteCommitTest.java | 12 +-
.../stage/write/WriteOptimisticVerifyTest.java | 35 +-
.../mvcc/stage/write/WriteStartTest.java | 26 +-
.../mvcc/stage/write/WriteUniqueVerifyIT.java | 34 +-
.../mvcc/stage/write/WriteUniqueVerifyTest.java | 13 +-
.../persistence/collection/rx/ParallelTest.java | 10 +-
.../serialization/EntityRepairImplTest.java | 147 -
.../impl/LogEntryIteratorTest.java | 131 -
.../impl/MinMaxLogEntryIteratorTest.java | 131 +
...MvccEntitySerializationStrategyImplTest.java | 167 +-
...cEntitySerializationStrategyProxyV1Test.java | 85 -
...ntitySerializationStrategyProxyV1_3Test.java | 81 +
...cEntitySerializationStrategyProxyV2Test.java | 83 -
...ntitySerializationStrategyProxyV2_3Test.java | 82 +
...ccEntitySerializationStrategyV1ImplTest.java | 78 +-
...ccEntitySerializationStrategyV2ImplTest.java | 73 +-
.../MvccEntitySerializationStrategyV2Test.java | 27 +-
...ccEntitySerializationStrategyV3ImplTest.java | 70 +
.../impl/MvccLESSTransientTest.java | 27 +-
...ccLogEntrySerializationStrategyImplTest.java | 235 +-
...EntrySerializationStrategyProxyImplTest.java | 90 +
...LogEntrySerializationStrategyV1ImplTest.java | 55 +
...LogEntrySerializationStrategyV2ImplTest.java | 53 +
.../impl/ScopedRowKeySerializerTest.java | 2 +-
.../impl/SerializationComparison.java | 4 +-
.../impl/SettingsValidationTest.java | 1 -
.../impl/UniqueFieldEntrySerializerTest.java | 108 +
...niqueValueSerializationStrategyImplTest.java | 344 ++
...ValueSerializationStrategyProxyImplTest.java | 100 +
...queValueSerializationStrategyV1ImplTest.java | 57 +
...queValueSerializationStrategyV2ImplTest.java | 58 +
...ctMvccEntityDataMigrationV1ToV3ImplTest.java | 214 ++
.../MvccEntityDataMigrationV1ToV3ImplTest.java | 99 +
.../MvccEntityDataMigrationV2ToV3ImplTest.java | 99 +
.../collection/util/EntityHelper.java | 1 -
.../collection/util/InvalidEntityGenerator.java | 1 +
.../collection/util/LogEntryMock.java | 116 +-
.../collection/util/UniqueValueEntryMock.java | 150 +
.../collection/util/VersionGenerator.java | 55 +
.../src/test/resources/log4j.properties | 1 +
stack/corepersistence/common/pom.xml | 21 +-
.../usergrid/persistence/core/CPManager.java | 28 +
.../persistence/core/astyanax/CassandraFig.java | 3 -
.../astyanax/DynamicCompositeParserImpl.java | 81 +
.../core/astyanax/FieldBufferBuilder.java | 15 +
.../core/astyanax/FieldBufferParser.java | 13 +
.../astyanax/MultiKeyColumnNameIterator.java | 4 +-
.../core/astyanax/MultiRowColumnIterator.java | 12 +-
.../core/executor/TaskExecutorFactory.java | 95 +
.../persistence/core/future/BetterFuture.java | 68 -
.../core/future/FutureObservable.java | 44 +
.../persistence/core/guice/CommonModule.java | 27 +-
.../persistence/core/guice/CurrentImpl.java | 42 -
.../persistence/core/guice/PreviousImpl.java | 42 -
.../persistence/core/guice/ProxyImpl.java | 42 -
.../core/hystrix/HystrixCassandra.java | 94 -
.../core/metrics/MetricsFactory.java | 2 +-
.../core/metrics/MetricsFactoryImpl.java | 44 +-
.../core/metrics/ObservableTimer.java | 72 +
.../migration/data/AbstractMigrationPlugin.java | 128 +
.../core/migration/data/DataMigration.java | 97 +-
.../migration/data/DataMigrationManager.java | 18 +-
.../data/DataMigrationManagerImpl.java | 237 +-
.../migration/data/MigrationDataProvider.java | 43 +
.../core/migration/data/MigrationInfoCache.java | 52 +
.../migration/data/MigrationInfoCacheImpl.java | 84 +
.../data/MigrationInfoSerialization.java | 28 +-
.../data/MigrationInfoSerializationImpl.java | 67 +-
.../core/migration/data/MigrationPlugin.java | 56 +
.../migration/data/MigrationRelationship.java | 117 +
.../core/migration/data/PluginPhase.java | 42 +
.../core/migration/data/ProgressObserver.java | 63 +
.../core/migration/data/VersionedData.java | 38 +
.../migration/data/VersionedMigrationSet.java | 164 +
.../core/migration/schema/Migration.java | 2 +-
.../persistence/core/rx/ObservableIterator.java | 2 +
.../persistence/core/rx/OrderedMerge.java | 1 -
.../persistence/core/rx/RxSchedulerFig.java | 60 +
.../persistence/core/rx/RxTaskScheduler.java | 38 +
.../core/rx/RxTaskSchedulerImpl.java | 131 +
.../core/scope/ApplicationScope.java | 9 +-
.../core/scope/ApplicationScopeImpl.java | 24 +-
.../core/task/NamedTaskExecutorImpl.java | 286 --
.../usergrid/persistence/core/task/Task.java | 48 -
.../persistence/core/task/TaskExecutor.java | 41 -
.../persistence/core/util/StringUtils.java | 34 +
.../MultiKeyColumnNameIteratorTest.java | 187 +-
.../astyanax/MultiRowColumnIteratorTest.java | 50 +-
.../persistence/core/astyanax/TestUtils.java | 2 +-
.../persistence/core/aws/NoAWSCredsRule.java | 98 +
.../core/guice/DataMigrationResetRule.java | 88 +
.../core/guice/MaxMigrationModule.java | 39 -
.../core/guice/MaxMigrationVersion.java | 40 -
.../core/guice/MigrationManagerRule.java | 7 +
.../core/guice/TestCommonModule.java | 1 +
.../persistence/core/guice/TestModule.java | 5 +-
.../data/DataMigrationManagerImplTest.java | 350 ++-
.../data/MigrationInfoSerializationTest.java | 32 +-
.../data/TestMigrationDataProvider.java | 61 +
.../migration/data/TestProgressObserver.java | 89 +
.../data/VersionedMigrationSetTest.java | 194 ++
.../core/task/NamedTaskExecutorImplTest.java | 271 --
.../persistence/core/util/IdGenerator.java | 51 +
.../apache/usergrid/persistence/graph/Edge.java | 4 +
.../usergrid/persistence/graph/GraphFig.java | 26 +-
.../persistence/graph/GraphManager.java | 36 +-
.../persistence/graph/GraphManagerFactory.java | 4 +-
.../usergrid/persistence/graph/MarkedEdge.java | 8 +-
.../persistence/graph/SearchByEdge.java | 6 +
.../persistence/graph/SearchByEdgeType.java | 8 +-
.../persistence/graph/guice/GraphModule.java | 115 +-
.../graph/impl/GraphManagerImpl.java | 652 ++--
.../persistence/graph/impl/SimpleEdge.java | 16 +-
.../graph/impl/SimpleMarkedEdge.java | 3 +
.../graph/impl/SimpleSearchByEdge.java | 37 +-
.../graph/impl/SimpleSearchByEdgeType.java | 51 +-
.../graph/impl/SimpleSearchByIdType.java | 12 +-
.../graph/impl/stage/EdgeDeleteListener.java | 2 +-
.../impl/stage/EdgeDeleteListenerImpl.java | 31 +-
.../graph/impl/stage/EdgeDeleteRepairImpl.java | 12 +-
.../graph/impl/stage/EdgeMetaRepair.java | 6 +-
.../graph/impl/stage/EdgeMetaRepairImpl.java | 195 +-
.../graph/impl/stage/NodeDeleteListener.java | 2 +-
.../impl/stage/NodeDeleteListenerImpl.java | 26 +-
.../EdgeMetadataSerialization.java | 3 +-
.../graph/serialization/EdgesObservable.java | 71 +
.../graph/serialization/TargetIdObservable.java | 38 +
.../EdgeMetadataSerializationProxyImpl.java | 161 +-
.../impl/EdgeMetadataSerializationV1Impl.java | 6 +
.../impl/EdgeMetadataSerializationV2Impl.java | 6 +
.../serialization/impl/EdgesObservableImpl.java | 129 +
.../serialization/impl/GraphDataVersions.java | 43 +
.../impl/GraphManagerFactoryImpl.java | 95 +
.../impl/NodeSerializationImpl.java | 36 +-
.../impl/TargetIdObservableImpl.java | 72 +
.../impl/migration/EdgeDataMigrationImpl.java | 136 +
.../impl/migration/GraphMigration.java | 53 +
.../impl/migration/GraphMigrationPlugin.java | 69 +
.../serialization/impl/migration/GraphNode.java | 39 +
.../impl/shard/DirectedEdgeMeta.java | 9 +-
.../shard/count/NodeShardApproximationImpl.java | 13 +-
.../NodeShardCounterSerializationImpl.java | 25 +-
.../shard/impl/NodeShardAllocationImpl.java | 16 +-
.../shard/impl/ShardGroupCompactionImpl.java | 154 +-
.../graph/CommittedGraphManagerIT.java | 135 -
.../persistence/graph/GraphManagerIT.java | 724 ++++-
.../persistence/graph/GraphManagerLoadTest.java | 19 +-
.../graph/GraphManagerShardConsistencyIT.java | 15 +-
.../graph/GraphManagerShardingIT.java | 13 +-
.../graph/GraphManagerStressTest.java | 19 +-
.../usergrid/persistence/graph/SimpleTest.java | 12 +-
.../graph/StorageGraphManagerIT.java | 240 --
.../graph/guice/TestGraphModule.java | 27 +-
.../graph/impl/EdgeDeleteListenerTest.java | 12 +-
.../graph/impl/NodeDeleteListenerTest.java | 10 +-
.../graph/impl/stage/EdgeDeleteRepairTest.java | 13 +-
.../graph/impl/stage/EdgeMetaRepairTest.java | 30 +-
.../EdgeMetaDataSerializationProxyV1Test.java | 15 +-
.../EdgeMetaDataSerializationProxyV2Test.java | 23 +-
.../EdgeMetaDataSerializationV1Test.java | 7 +-
.../EdgeMetaDataSerializationV2Test.java | 7 +-
.../EdgeMetadataSerializationTest.java | 39 +-
.../EdgeSerializationChopTest.java | 7 +-
.../serialization/EdgeSerializationTest.java | 21 +-
.../serialization/NodeSerializationTest.java | 15 +-
.../migration/EdgeDataMigrationImplTest.java | 177 ++
.../impl/shard/EdgeShardSerializationTest.java | 7 +-
.../impl/shard/NodeShardAllocationTest.java | 27 +-
.../impl/shard/NodeShardCacheTest.java | 7 +-
.../impl/shard/ShardGroupCompactionTest.java | 11 +-
.../shard/count/NodeShardApproximationTest.java | 9 +-
.../NodeShardCounterSerializationTest.java | 6 +-
.../shard/impl/ShardEntryGroupIteratorTest.java | 15 +-
...rceDirectedEdgeDescendingComparatorTest.java | 23 +-
...getDirectedEdgeDescendingComparatorTest.java | 23 +-
.../graph/test/util/EdgeTestUtils.java | 63 +-
.../usergrid/persistence/map/MapManager.java | 25 +-
.../persistence/map/guice/MapModule.java | 8 +-
.../persistence/map/guice/TestMapModule.java | 3 +-
stack/corepersistence/model/pom.xml | 1 -
.../persistence/model/entity/Entity.java | 23 +-
.../persistence/model/entity/EntityMap.java | 80 +
.../model/entity/EntityToMapConverter.java | 160 +
.../usergrid/persistence/model/entity/Id.java | 3 +
.../model/entity/MapToEntityConverter.java | 203 ++
.../persistence/model/entity/SimpleId.java | 36 +-
.../persistence/model/field/AbstractField.java | 8 +
.../persistence/model/field/ListField.java | 8 +-
.../persistence/model/field/StringField.java | 20 +
.../persistence/model/util/EntityUtils.java | 72 +
stack/corepersistence/pom.xml | 38 +-
stack/corepersistence/queryindex/pom.xml | 63 +-
.../index/query/tree/CpQueryFilter.g | 20 +-
.../persistence/index/AliasedEntityIndex.java | 13 +-
.../index/ApplicationEntityIndex.java | 53 +
.../persistence/index/CandidateResult.java | 84 +
.../persistence/index/CandidateResults.java | 103 +
.../usergrid/persistence/index/EntityIndex.java | 75 +-
.../persistence/index/EntityIndexBatch.java | 28 +-
.../persistence/index/EntityIndexFactory.java | 11 +-
.../persistence/index/IndexBatchBuffer.java | 36 -
.../persistence/index/IndexBufferConsumer.java | 37 -
.../persistence/index/IndexBufferProducer.java | 37 -
.../usergrid/persistence/index/IndexEdge.java | 36 +
.../usergrid/persistence/index/IndexFig.java | 129 +-
.../persistence/index/IndexIdentifier.java | 92 -
.../index/IndexOperationMessage.java | 138 -
.../persistence/index/IndexRefreshCommand.java | 50 +
.../usergrid/persistence/index/IndexScope.java | 38 -
.../usergrid/persistence/index/SearchEdge.java | 61 +
.../usergrid/persistence/index/SearchType.java | 51 +
.../usergrid/persistence/index/SearchTypes.java | 46 +-
.../persistence/index/SelectFieldMapping.java | 58 +
.../persistence/index/guice/IndexModule.java | 44 +-
.../persistence/index/guice/QueueProvider.java | 116 -
.../persistence/index/impl/BatchOperation.java | 41 +
.../persistence/index/impl/BatchRequest.java | 41 -
.../persistence/index/impl/BufferQueue.java | 68 -
.../index/impl/BufferQueueInMemoryImpl.java | 116 -
.../index/impl/BufferQueueSQSImpl.java | 307 --
.../index/impl/DeIndexOperation.java | 104 +
.../persistence/index/impl/DeIndexRequest.java | 115 -
.../persistence/index/impl/EntityField.java | 148 +
.../index/impl/EntityMappingParser.java | 247 ++
.../index/impl/EntityToMapConverter.java | 108 +
.../impl/EsApplicationEntityIndexImpl.java | 259 ++
.../index/impl/EsEntityIndexBatchImpl.java | 292 +-
.../index/impl/EsEntityIndexFactoryImpl.java | 44 +-
.../index/impl/EsEntityIndexImpl.java | 604 +---
.../index/impl/EsIndexBufferConsumerImpl.java | 347 +--
.../index/impl/EsIndexBufferProducerImpl.java | 65 -
.../persistence/index/impl/EsIndexCache.java | 142 -
.../index/impl/EsIndexCacheImpl.java | 141 +
.../persistence/index/impl/EsProvider.java | 129 +-
.../persistence/index/impl/EsQueryVistor.java | 589 ++--
.../persistence/index/impl/FieldParser.java | 45 +
.../index/impl/FlushBufferQueue.java | 23 +
.../persistence/index/impl/GeoSortFields.java | 95 +
.../persistence/index/impl/IndexAlias.java | 42 +
.../index/impl/IndexBufferConsumer.java | 38 +
.../persistence/index/impl/IndexCache.java | 43 +
.../persistence/index/impl/IndexEdgeImpl.java | 84 +
.../persistence/index/impl/IndexIdentifier.java | 46 +
.../index/impl/IndexIdentifierImpl.java | 67 +
.../persistence/index/impl/IndexOperation.java | 107 +
.../index/impl/IndexOperationMessage.java | 121 +
.../index/impl/IndexRefreshCommandImpl.java | 205 ++
.../persistence/index/impl/IndexRequest.java | 125 -
.../persistence/index/impl/IndexScopeImpl.java | 88 -
.../persistence/index/impl/IndexingUtils.java | 265 +-
.../index/impl/NoOpFilterBuilder.java | 45 +
.../index/impl/NoOpQueryBuilder.java | 45 +
.../persistence/index/impl/SearchEdgeImpl.java | 120 +
.../impl/SearchRequestBuilderStrategy.java | 290 ++
.../persistence/index/impl/SortBuilder.java | 44 +
.../index/migration/EsIndexMigrationPlugin.java | 46 +
.../index/migration/IndexDataVersions.java | 39 +
.../index/migration/IndexMigration.java | 35 +
.../index/migration/LegacyIndexIdentifier.java | 78 +
.../index/query/CandidateResult.java | 74 -
.../index/query/CandidateResults.java | 90 -
.../persistence/index/query/EntityResults.java | 108 -
.../persistence/index/query/ParsedQuery.java | 205 ++
.../index/query/ParsedQueryBuilder.java | 108 +
.../usergrid/persistence/index/query/Query.java | 1428 ---------
.../persistence/index/query/Results.java | 148 -
.../persistence/index/query/SortPredicate.java | 144 +
.../index/query/tree/QueryVisitor.java | 55 +-
.../index/utils/IndexValidationUtils.java | 24 +-
.../persistence/index/utils/ListUtils.java | 1 +
.../persistence/index/utils/StringUtils.java | 62 -
.../persistence/index/usergrid-mappings.json | 99 +
.../persistence/index/guice/IndexTestFig.java | 62 +
.../index/guice/TestIndexModule.java | 42 +-
.../index/impl/BufferQueueSQSImplTest.java | 169 -
.../index/impl/CorePerformanceIT.java | 339 --
.../index/impl/ElasticSearchRule.java | 58 +
.../impl/EntityConnectionIndexImplTest.java | 306 --
.../index/impl/EntityIndexMapUtils.java | 57 +-
.../persistence/index/impl/EntityIndexTest.java | 1096 +++++--
.../index/impl/EntityToMapConverterTest.java | 653 ++++
.../persistence/index/impl/GeoPagingTest.java | 163 +
.../index/impl/IndexLoadTestsIT.java | 393 +++
.../persistence/query/tree/GrammarTreeTest.java | 157 +-
.../src/test/resources/dynamic-test.properties | 2 +-
.../src/test/resources/log4j.properties | 10 +-
.../src/test/resources/usergrid-UNIT.properties | 7 +
.../usergrid/persistence/queue/QueueFig.java | 8 +
.../persistence/queue/NoAWSCredsRule.java | 98 -
.../persistence/queue/QueueManagerTest.java | 3 +-
.../queue/guice/TestQueueModule.java | 3 +-
stack/java-sdk-old/pom.xml | 102 -
.../scenarios/ConnectionScenarios.scala | 2 +-
.../usergrid/mongo/protocol/OpDelete.java | 4 +-
.../apache/usergrid/mongo/protocol/OpQuery.java | 6 +-
.../usergrid/mongo/protocol/OpUpdate.java | 4 +-
.../apache/usergrid/mongo/BasicMongoTest.java | 8 +-
.../apache/usergrid/mongo/MongoQueryTest.java | 18 +-
stack/pom.xml | 29 +-
stack/rest/pom.xml | 17 +-
.../usergrid/rest/AbstractContextResource.java | 102 +-
.../org/apache/usergrid/rest/ApiResponse.java | 40 +-
.../org/apache/usergrid/rest/IndexResource.java | 12 +-
.../apache/usergrid/rest/MigrateResource.java | 122 +-
.../org/apache/usergrid/rest/RootResource.java | 19 +-
.../rest/applications/ApplicationResource.java | 38 +-
.../rest/applications/ServiceResource.java | 43 +-
.../notifiers/NotifiersResource.java | 11 +-
.../rest/applications/users/UsersResource.java | 4 +-
.../exceptions/AbstractExceptionMapper.java | 11 +-
.../exceptions/ConflictExceptionMapper.java | 35 +
.../exceptions/NotFoundExceptionMapper.java | 35 +
.../rest/exceptions/UncaughtException.java | 36 +
.../rest/management/ManagementResource.java | 59 +-
.../organizations/OrganizationResource.java | 28 +-
.../organizations/OrganizationsResource.java | 23 +-
.../applications/ApplicationResource.java | 61 +-
.../applications/ApplicationsResource.java | 34 +-
.../imports/FileIncludesResource.java | 9 -
.../applications/imports/ImportsResource.java | 83 +-
.../organizations/OrganizationResource.java | 32 +-
.../OAuth2AccessTokenSecurityFilter.java | 2 +-
.../rest/test/RefreshIndexResource.java | 25 +-
.../java/org/apache/usergrid/rest/BasicIT.java | 302 +-
.../apache/usergrid/rest/IndexResourceIT.java | 1 +
.../apache/usergrid/rest/NotificationsIT.java | 93 +-
.../apache/usergrid/rest/SystemResourceIT.java | 10 +-
.../rest/applications/ApplicationCreateIT.java | 135 +
.../rest/applications/ApplicationDeleteIT.java | 389 +++
.../applications/ApplicationDeleteTest.java | 54 -
.../applications/ApplicationResourceIT.java | 100 +-
.../collection/CollectionsResourceIT.java | 19 +-
.../collection/paging/PagingResourceIT.java | 17 +-
.../collection/users/OwnershipResourceIT.java | 4 +-
.../collection/users/PermissionsResourceIT.java | 15 -
.../collection/users/RetrieveUsersTest.java | 4 +-
.../collection/users/UserResourceIT.java | 16 +-
.../events/ApplicationRequestCounterIT.java | 2 +-
.../applications/queries/BasicGeoTests.java | 21 +-
.../applications/queries/GeoPagingTest.java | 58 +-
.../applications/queries/MatrixQueryTests.java | 298 +-
.../rest/applications/queries/OrderByTest.java | 2 +-
.../queues/AbstractQueueResourceIT.java | 484 ---
.../queues/QueueResourceLong1IT.java | 149 -
.../queues/QueueResourceLong2IT.java | 99 -
.../queues/QueueResourceLong3IT.java | 201 --
.../queues/QueueResourceShortIT.java | 432 ---
.../rest/filters/ContentTypeResourceIT.java | 55 +-
.../usergrid/rest/management/AccessTokenIT.java | 418 +--
.../usergrid/rest/management/AdminUsersIT.java | 64 +-
.../rest/management/ExportResourceIT.java | 674 ++--
.../rest/management/ImportResourceIT.java | 179 +-
.../rest/management/ManagementResourceIT.java | 151 +-
.../rest/management/OrganizationsIT.java | 82 +-
.../organizations/AdminEmailEncodingIT.java | 3 +-
.../test/resource2point0/AbstractRestIT.java | 4 +-
.../rest/test/resource2point0/ClientSetup.java | 30 +-
.../rest/test/resource2point0/README.md | 111 +
.../rest/test/resource2point0/RestClient.java | 1 +
.../endpoints/ApplicationsResource.java | 34 +-
.../endpoints/CollectionEndpoint.java | 13 +
.../endpoints/NamedResource.java | 158 +-
.../endpoints/OrganizationResource.java | 6 +-
.../resource2point0/endpoints/RootResource.java | 1 +
.../endpoints/SetupResource.java | 6 +
.../endpoints/mgmt/ApplicationResource.java | 54 +-
.../endpoints/mgmt/ApplicationsResource.java | 58 +
.../endpoints/mgmt/ManagementResource.java | 4 +
.../endpoints/mgmt/ManagementResponse.java | 72 +
.../endpoints/mgmt/MeResource.java | 32 +
.../mgmt/OrganizationApplicationResponse.java | 60 +
.../endpoints/mgmt/OrganizationResource.java | 5 +-
.../endpoints/mgmt/PasswordResource.java | 16 -
.../endpoints/mgmt/RevokeTokenResource.java | 32 +
.../endpoints/mgmt/RevokeTokensResource.java | 33 +
.../endpoints/mgmt/TokenResource.java | 25 +-
.../endpoints/mgmt/UserResource.java | 8 +
.../endpoints/mgmt/UsersResource.java | 15 -
stack/rest/src/test/resources/log4j.properties | 10 +-
.../resources/testImportCorrect.testCol.1.json | 18 -
...testImportInvalidJson.testApplication.3.json | 153 -
.../testimport-bad-json-testapp.3.json | 153 +
.../src/test/resources/testimport-bad-json.json | 67 +
.../resources/testimport-correct-testcol.1.json | 18 +
.../test/resources/testimport-test-app.2.json | 56 +
.../resources/usergrid-custom-test.properties | 2 +
stack/services/pom.xml | 51 +-
.../usergrid/management/ApplicationInfo.java | 4 +
.../usergrid/management/ManagementService.java | 10 +-
.../cassandra/ApplicationCreatorImpl.java | 12 +-
.../cassandra/ManagementServiceImpl.java | 235 +-
.../management/export/ExportServiceImpl.java | 6 +-
.../management/importer/FileImportJob.java | 8 +-
.../management/importer/FileImportTracker.java | 6 +-
.../usergrid/management/importer/ImportJob.java | 4 +-
.../management/importer/ImportServiceImpl.java | 82 +-
.../usergrid/security/AuthPrincipalType.java | 3 +-
.../security/providers/AbstractProvider.java | 1 +
.../security/providers/FacebookProvider.java | 6 +-
.../security/providers/FoursquareProvider.java | 6 +-
.../shiro/PrincipalCredentialsToken.java | 20 +-
.../apache/usergrid/security/shiro/Realm.java | 2 +-
.../services/AbstractCollectionService.java | 17 +-
.../services/AbstractConnectionsService.java | 4 +-
.../AbstractPathBasedColllectionService.java | 2 +-
.../usergrid/services/AbstractService.java | 2 +-
.../usergrid/services/ServiceContext.java | 2 +-
.../services/ServiceManagerFactory.java | 4 -
.../usergrid/services/ServiceParameter.java | 3 +-
.../usergrid/services/ServiceRequest.java | 2 +-
.../usergrid/services/ServiceResults.java | 38 +-
.../applications/ApplicationsService.java | 2 +-
.../services/devices/DevicesService.java | 2 +-
.../usergrid/services/groups/GroupsService.java | 2 +-
.../groups/activities/ActivitiesService.java | 2 +-
.../users/activities/ActivitiesService.java | 4 +-
.../notifications/InactiveDeviceManager.java | 8 +-
.../notifications/NotificationsService.java | 4 +-
.../apns/FailedConnectionListener.java | 31 +-
.../impl/ApplicationQueueManagerImpl.java | 197 +-
.../services/notifications/wns/WNSAdapter.java | 60 +-
.../usergrid/services/roles/RolesService.java | 10 +-
.../usergrid/services/users/UsersService.java | 2 +-
.../users/activities/ActivitiesService.java | 4 +-
.../org/apache/usergrid/ServiceApplication.java | 18 +-
.../org/apache/usergrid/ServiceITSetup.java | 5 +
.../org/apache/usergrid/ServiceITSetupImpl.java | 43 +-
.../migration/AppInfoMigrationPluginTest.java | 231 ++
.../apache/usergrid/management/EmailFlowIT.java | 18 +-
.../usergrid/management/OrganizationIT.java | 16 +-
.../org/apache/usergrid/management/RoleIT.java | 4 +-
.../cassandra/ApplicationCreatorIT.java | 14 +-
.../cassandra/ManagementServiceIT.java | 35 +-
.../management/export/ExportServiceIT.java | 20 +-
.../importer/FileImportTrackerTest.java | 12 +-
.../management/importer/ImportCollectionIT.java | 20 +-
.../importer/ImportConnectionsTest.java | 9 +-
.../management/importer/ImportServiceIT.java | 2 +-
.../security/tokens/TokenServiceIT.java | 2 -
.../usergrid/services/ActivitiesServiceIT.java | 2 +-
.../usergrid/services/CollectionServiceIT.java | 2 +
.../usergrid/services/ConnectionsServiceIT.java | 10 +-
.../usergrid/services/GroupServiceIT.java | 6 +-
.../usergrid/services/RolesServiceIT.java | 4 +-
.../usergrid/services/ServiceFactoryIT.java | 8 +-
.../usergrid/services/ServiceInvocationIT.java | 2 +-
.../usergrid/services/ServiceRequestIT.java | 9 +-
.../usergrid/services/UsersServiceIT.java | 2 +-
.../AbstractServiceNotificationIT.java | 4 +-
.../notifications/NotifiersServiceIT.java | 19 +-
.../apns/NotificationsServiceIT.java | 66 +-
.../gcm/NotificationsServiceIT.java | 4 +-
.../src/test/resources/log4j.properties | 6 +-
.../resources/usergrid-custom-test.properties | 5 +
.../usergrid/cassandra/SpringResource.java | 13 +-
.../setup/ConcurrentProcessSingleton.java | 49 +-
.../org/apache/usergrid/tools/AppAudit.java | 3 +-
.../org/apache/usergrid/tools/CredExport.java | 2 +-
.../org/apache/usergrid/tools/DupOrgRepair.java | 1 -
.../java/org/apache/usergrid/tools/Import.java | 14 +-
.../usergrid/tools/UniqueIndexCleanup.java | 6 +-
876 files changed, 37842 insertions(+), 44603 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ed8ee9a9/stack/rest/src/main/java/org/apache/usergrid/rest/ApiResponse.java
----------------------------------------------------------------------
diff --cc stack/rest/src/main/java/org/apache/usergrid/rest/ApiResponse.java
index 93e7f84,590dabf..7a9e262
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/ApiResponse.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/ApiResponse.java
@@@ -23,18 -23,8 +23,20 @@@ import com.fasterxml.jackson.annotation
import com.fasterxml.jackson.annotation.JsonPropertyOrder;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import com.fasterxml.jackson.databind.annotation.JsonSerialize.Inclusion;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.UUID;
+
+import javax.xml.bind.annotation.XmlAnyElement;
+import javax.xml.bind.annotation.XmlRootElement;
+import org.springframework.beans.factory.annotation.Autowired;
+ import org.apache.commons.lang.ClassUtils;
+ import org.apache.commons.lang.StringUtils;
import org.apache.usergrid.persistence.AggregateCounterSet;
import org.apache.usergrid.persistence.Entity;
import org.apache.usergrid.persistence.Results;
[08/22] incubator-usergrid git commit: fix connections
Posted by gr...@apache.org.
fix connections
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/0c16f0e5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/0c16f0e5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/0c16f0e5
Branch: refs/heads/USERGRID-608
Commit: 0c16f0e5a0655100dd875beae63574eec3218207
Parents: 5b1dfa1
Author: Shawn Feldman <sf...@apache.org>
Authored: Thu May 14 16:30:22 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Thu May 14 16:30:22 2015 -0600
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 2 +-
.../corepersistence/CpRelationManager.java | 39 ++++++--------------
.../corepersistence/index/IndexServiceImpl.java | 3 +-
.../usergrid/persistence/RelationManager.java | 2 -
.../serialization/impl/EdgesObservableImpl.java | 2 +
.../usergrid/management/OrganizationIT.java | 8 ++--
.../resources/usergrid-custom-test.properties | 1 +
7 files changed, 21 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0c16f0e5/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 63018cb..3f615b4 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -2622,7 +2622,7 @@ public class CpEntityManager implements EntityManager {
String collectionName = Schema.defaultCollectionName( eType );
CpRelationManager cpr = ( CpRelationManager ) getRelationManager( getApplication() );
- cpr.addToCollection( collectionName, entity, cpEntity, false );
+ cpr.addToCollection( collectionName, entity);
// Invoke counters
incrementEntityCollection( collectionName, timestamp );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0c16f0e5/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index 4993d88..797788e 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -20,6 +20,7 @@ package org.apache.usergrid.corepersistence;
import java.util.*;
import org.apache.usergrid.persistence.graph.*;
+import org.apache.usergrid.utils.InflectionUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.Assert;
@@ -343,22 +344,10 @@ public class CpRelationManager implements RelationManager {
return null;
}
- return addToCollection( collectionName, itemRef, ( collection != null && collection.getLinkedCollection() != null ) );
- }
-
-
- public Entity addToCollection( String collectionName, EntityRef itemRef, boolean connectBack ) throws Exception {
Id entityId = new SimpleId( itemRef.getUuid(), itemRef.getType() );
- org.apache.usergrid.persistence.model.entity.Entity memberEntity = ( ( CpEntityManager ) em ).load( entityId );
-
- return addToCollection( collectionName, itemRef, memberEntity, connectBack );
- }
-
+ org.apache.usergrid.persistence.model.entity.Entity memberEntity = ( ( CpEntityManager ) em ).load(entityId);
- public Entity addToCollection( final String collectionName, final EntityRef itemRef,
- final org.apache.usergrid.persistence.model.entity.Entity memberEntity,
- final boolean connectBack ) throws Exception {
// don't fetch entity if we've already got one
final Entity itemEntity;
@@ -373,10 +362,6 @@ public class CpRelationManager implements RelationManager {
return null;
}
- CollectionInfo collection = getDefaultSchema().getCollection( headEntity.getType(), collectionName );
- if ( ( collection != null ) && !collection.getType().equals( itemRef.getType() ) ) {
- return null;
- }
if ( memberEntity == null ) {
@@ -393,8 +378,10 @@ public class CpRelationManager implements RelationManager {
// create graph edge connection from head entity to member entity
final Edge edge = createCollectionEdge( cpHeadEntity.getId(), collectionName, memberEntity.getId() );
+
GraphManager gm = managerCache.getGraphManager( applicationScope );
gm.writeEdge( edge ).toBlocking().last();
+ //reverse
//perform indexing
@@ -404,6 +391,13 @@ public class CpRelationManager implements RelationManager {
}
indexService.queueNewEdge(applicationScope, memberEntity, edge);
+ //reverse
+ if(!cpHeadEntity.getId().getType().equals("application")) {
+ String pluralType = InflectionUtils.pluralize(cpHeadEntity.getId().getType());
+ final Edge reverseEdge = createCollectionEdge(memberEntity.getId(), pluralType, cpHeadEntity.getId());
+ gm.writeEdge(reverseEdge).toBlocking().last();
+ indexService.queueNewEdge(applicationScope, cpHeadEntity, reverseEdge);
+ }
if ( logger.isDebugEnabled() ) {
@@ -417,17 +411,6 @@ public class CpRelationManager implements RelationManager {
}
- @Override
- public Entity addToCollections( List<EntityRef> owners, String collectionName ) throws Exception {
-
- // TODO: this addToCollections() implementation seems wrong.
- for ( EntityRef eref : owners ) {
- addToCollection( collectionName, eref );
- }
-
- return null;
- }
-
@Override
public Entity createItemInCollection( String collectionName, String itemType, Map<String, Object> properties )
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0c16f0e5/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
index 1fc8eea..9bb0728 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
@@ -138,10 +138,11 @@ public class IndexServiceImpl implements IndexService {
//if the node is the target node, generate our scope correctly
if ( edge.getTargetNode().equals( entity.getId() ) ) {
+
return generateScopeFromSource( edge );
}
- return generateScopeFromTarget( edge );
+ throw new IllegalArgumentException("target not equal to entity + "+entity.getId());
} ).flatMap( indexEdge -> {
final ApplicationEntityIndex ei = entityIndexFactory.createApplicationEntityIndex( applicationScope );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0c16f0e5/stack/core/src/main/java/org/apache/usergrid/persistence/RelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/RelationManager.java b/stack/core/src/main/java/org/apache/usergrid/persistence/RelationManager.java
index 0b2fca8..0011183 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/RelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/RelationManager.java
@@ -54,8 +54,6 @@ public interface RelationManager {
public Entity addToCollection( String collectionName, EntityRef itemRef ) throws Exception;
- public Entity addToCollections( List<EntityRef> owners, String collectionName ) throws Exception;
-
public Entity createItemInCollection( String collectionName, String itemType, Map<String, Object> properties )
throws Exception;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0c16f0e5/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgesObservableImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgesObservableImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgesObservableImpl.java
index df9e094..55d9d7d 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgesObservableImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgesObservableImpl.java
@@ -126,4 +126,6 @@ public class EdgesObservableImpl implements EdgesObservable {
Optional.<Edge>absent() ) );
} );
}
+
+
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0c16f0e5/stack/services/src/test/java/org/apache/usergrid/management/OrganizationIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/management/OrganizationIT.java b/stack/services/src/test/java/org/apache/usergrid/management/OrganizationIT.java
index cbd0e6d..3c93fa3 100644
--- a/stack/services/src/test/java/org/apache/usergrid/management/OrganizationIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/management/OrganizationIT.java
@@ -73,14 +73,14 @@ public class OrganizationIT {
setup.getEntityIndex().refresh();
Map<UUID, String> userOrganizations = setup.getMgmtSvc().getOrganizationsForAdminUser(
organization.getOwner().getUuid() );
- assertEquals( "wrong number of organizations", 1, userOrganizations.size() );
+ assertEquals("wrong number of organizations", 1, userOrganizations.size());
List<UserInfo> users = setup.getMgmtSvc().getAdminUsersForOrganization(
- organization.getOrganization().getUuid() );
- assertEquals( "wrong number of users", 1, users.size() );
+ organization.getOrganization().getUuid());
+ assertEquals("wrong number of users", 1, users.size());
ApplicationInfo applicationInfo = setup.getMgmtSvc().createApplication(organization.getOrganization().getUuid(), "ed-application");
- assertNotNull( applicationInfo.getId() );
+ assertNotNull(applicationInfo.getId());
setup.getEntityIndex().refresh();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0c16f0e5/stack/services/src/test/resources/usergrid-custom-test.properties
----------------------------------------------------------------------
diff --git a/stack/services/src/test/resources/usergrid-custom-test.properties b/stack/services/src/test/resources/usergrid-custom-test.properties
index b5f86ae..d01231f 100644
--- a/stack/services/src/test/resources/usergrid-custom-test.properties
+++ b/stack/services/src/test/resources/usergrid-custom-test.properties
@@ -32,6 +32,7 @@ usergrid.notifications.listener.run=false
elasticsearch.index_prefix=services_tests
elasticsearch.buffer_timeout=1
+elasticsearch.queue_impl.resolution=true
[21/22] incubator-usergrid git commit: [USERGRID-642] Remove password
properties from user objects in portal
Posted by gr...@apache.org.
[USERGRID-642] Remove password properties from user objects in portal
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/53563e83
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/53563e83
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/53563e83
Branch: refs/heads/USERGRID-608
Commit: 53563e83b45b85c30a756530f366b6312f6d45ac
Parents: b28aeee
Author: ryan bridges <ry...@apache.org>
Authored: Thu May 21 12:17:01 2015 -0400
Committer: ryan bridges <ry...@apache.org>
Committed: Thu May 21 12:17:01 2015 -0400
----------------------------------------------------------------------
portal/js/users/users-controller.js | 19 +++++++++++++++++--
1 file changed, 17 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/53563e83/portal/js/users/users-controller.js
----------------------------------------------------------------------
diff --git a/portal/js/users/users-controller.js b/portal/js/users/users-controller.js
index fa489f7..3978357 100644
--- a/portal/js/users/users-controller.js
+++ b/portal/js/users/users-controller.js
@@ -61,15 +61,29 @@ AppServices.Controllers.controller('UsersCtrl', ['ug', '$scope', '$rootScope', '
clearNewUserForm();
break;
}
- }
+ };
ug.getUsers();
+ function sanitizeUsersCollection(){
+ var cleanProperties = ["password", "oldpassword", "newpassword"]
+ if($scope.usersCollection._list.length > 0){
+ $scope.usersCollection._list.forEach(function(user, i){
+ Object.keys(user._data).forEach(function(key){
+ if(cleanProperties.indexOf(key.toLowerCase()) !== -1){
+ // console.warn("Removing %s from %s", key, user._data.uuid);
+ delete user._data[key]
+ }
+ });
+ $scope.usersCollection._list[i]=user;
+ })
+ }
+ }
$scope.$on('users-received', function(event, users) {
$scope.usersCollection = users;
$scope.usersSelected = false;
$scope.hasUsers = users._list.length;
-
+ sanitizeUsersCollection();
if(users._list.length > 0){
$scope.selectUser(users._list[0]._data.uuid)
}
@@ -79,6 +93,7 @@ AppServices.Controllers.controller('UsersCtrl', ['ug', '$scope', '$rootScope', '
});
$scope.$on('users-create-success', function () {
+ sanitizeUsersCollection();
$rootScope.$broadcast("alert", "success", "User successfully created.");
});
[05/22] incubator-usergrid git commit: rename vars
Posted by gr...@apache.org.
rename vars
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/e3ba2b0b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/e3ba2b0b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/e3ba2b0b
Branch: refs/heads/USERGRID-608
Commit: e3ba2b0bca060b59c854e82502bade67ce01ae77
Parents: f5cb788
Author: Shawn Feldman <sf...@apache.org>
Authored: Wed May 13 10:35:18 2015 -0700
Committer: Shawn Feldman <sf...@apache.org>
Committed: Wed May 13 10:35:18 2015 -0700
----------------------------------------------------------------------
.../corepersistence/CpRelationManager.java | 77 ++++++++++----------
.../cassandra/ManagementServiceImpl.java | 3 +-
2 files changed, 38 insertions(+), 42 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e3ba2b0b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index fb66b97..bd3d137 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -19,6 +19,8 @@ package org.apache.usergrid.corepersistence;
import java.util.*;
+import org.apache.usergrid.persistence.graph.*;
+import org.apache.usergrid.utils.UUIDUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.Assert;
@@ -48,10 +50,6 @@ import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.entities.Group;
import org.apache.usergrid.persistence.entities.User;
-import org.apache.usergrid.persistence.graph.Edge;
-import org.apache.usergrid.persistence.graph.GraphManager;
-import org.apache.usergrid.persistence.graph.SearchByEdge;
-import org.apache.usergrid.persistence.graph.SearchByEdgeType;
import org.apache.usergrid.persistence.graph.impl.SimpleEdge;
import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdge;
import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdgeType;
@@ -270,11 +268,11 @@ public class CpRelationManager implements RelationManager {
@SuppressWarnings( "unchecked" )
@Override
- public boolean isCollectionMember( String collName, EntityRef entity ) throws Exception {
+ public boolean isCollectionMember( String collectionName, EntityRef entity ) throws Exception {
Id entityId = new SimpleId( entity.getUuid(), entity.getType() );
- String edgeType = CpNamingUtils.getEdgeTypeFromCollectionName(collName);
+ String edgeType = CpNamingUtils.getEdgeTypeFromCollectionName(collectionName);
logger.debug( "isCollectionMember(): Checking for edge type {} from {}:{} to {}:{}", new Object[] {
edgeType, headEntity.getType(), headEntity.getUuid(), entity.getType(), entity.getUuid()
@@ -313,7 +311,6 @@ public class CpRelationManager implements RelationManager {
public Results getCollection( String collectionName, UUID startResult, int count, Level resultsLevel,
boolean reversed ) throws Exception {
-
final String ql;
if ( startResult != null ) {
@@ -332,35 +329,35 @@ public class CpRelationManager implements RelationManager {
@Override
- public Results getCollection( String collName, Query query, Level level ) throws Exception {
+ public Results getCollection( String collectionName, Query query, Level level ) throws Exception {
- return searchCollection( collName, query );
+ return searchCollection( collectionName, query );
}
// add to a named collection of the head entity
@Override
- public Entity addToCollection( String collName, EntityRef itemRef ) throws Exception {
+ public Entity addToCollection( String collectionName, EntityRef itemRef ) throws Exception {
- CollectionInfo collection = getDefaultSchema().getCollection( headEntity.getType(), collName );
+ CollectionInfo collection = getDefaultSchema().getCollection( headEntity.getType(), collectionName );
if ( ( collection != null ) && !collection.getType().equals( itemRef.getType() ) ) {
return null;
}
- return addToCollection( collName, itemRef, ( collection != null && collection.getLinkedCollection() != null ) );
+ return addToCollection( collectionName, itemRef, ( collection != null && collection.getLinkedCollection() != null ) );
}
- public Entity addToCollection( String collName, EntityRef itemRef, boolean connectBack ) throws Exception {
+ public Entity addToCollection( String collectionName, EntityRef itemRef, boolean connectBack ) throws Exception {
Id entityId = new SimpleId( itemRef.getUuid(), itemRef.getType() );
org.apache.usergrid.persistence.model.entity.Entity memberEntity = ( ( CpEntityManager ) em ).load( entityId );
- return addToCollection( collName, itemRef, memberEntity, connectBack );
+ return addToCollection( collectionName, itemRef, memberEntity, connectBack );
}
- public Entity addToCollection( final String collName, final EntityRef itemRef,
+ public Entity addToCollection( final String collectionName, final EntityRef itemRef,
final org.apache.usergrid.persistence.model.entity.Entity memberEntity,
final boolean connectBack ) throws Exception {
@@ -377,7 +374,7 @@ public class CpRelationManager implements RelationManager {
return null;
}
- CollectionInfo collection = getDefaultSchema().getCollection( headEntity.getType(), collName );
+ CollectionInfo collection = getDefaultSchema().getCollection( headEntity.getType(), collectionName );
if ( ( collection != null ) && !collection.getType().equals( itemRef.getType() ) ) {
return null;
}
@@ -396,7 +393,7 @@ public class CpRelationManager implements RelationManager {
// create graph edge connection from head entity to member entity
- final Edge edge = createCollectionEdge( cpHeadEntity.getId(), collName, memberEntity.getId() );
+ final Edge edge = createCollectionEdge( cpHeadEntity.getId(), collectionName, memberEntity.getId() );
GraphManager gm = managerCache.getGraphManager( applicationScope );
gm.writeEdge( edge ).toBlocking().last();
@@ -412,7 +409,7 @@ public class CpRelationManager implements RelationManager {
if ( logger.isDebugEnabled() ) {
logger.debug( "Added entity {}:{} to collection {}", new Object[] {
- itemRef.getUuid().toString(), itemRef.getType(), collName
+ itemRef.getUuid().toString(), itemRef.getType(), collectionName
} );
}
@@ -422,11 +419,11 @@ public class CpRelationManager implements RelationManager {
@Override
- public Entity addToCollections( List<EntityRef> owners, String collName ) throws Exception {
+ public Entity addToCollections( List<EntityRef> owners, String collectionName ) throws Exception {
// TODO: this addToCollections() implementation seems wrong.
for ( EntityRef eref : owners ) {
- addToCollection( collName, eref );
+ addToCollection( collectionName, eref );
}
return null;
@@ -434,12 +431,12 @@ public class CpRelationManager implements RelationManager {
@Override
- public Entity createItemInCollection( String collName, String itemType, Map<String, Object> properties )
+ public Entity createItemInCollection( String collectionName, String itemType, Map<String, Object> properties )
throws Exception {
if ( headEntity.getUuid().equals( applicationId ) ) {
if ( itemType.equals( TYPE_ENTITY ) ) {
- itemType = singularize( collName );
+ itemType = singularize( collectionName );
}
if ( itemType.equals( TYPE_ROLE ) ) {
@@ -453,13 +450,13 @@ public class CpRelationManager implements RelationManager {
return em.create( itemType, properties );
}
- else if ( headEntity.getType().equals( Group.ENTITY_TYPE ) && ( collName.equals( COLLECTION_ROLES ) ) ) {
+ else if ( headEntity.getType().equals( Group.ENTITY_TYPE ) && ( collectionName.equals( COLLECTION_ROLES ) ) ) {
UUID groupId = headEntity.getUuid();
String roleName = ( String ) properties.get( PROPERTY_NAME );
return em.createGroupRole( groupId, roleName, ( Long ) properties.get( PROPERTY_INACTIVITY ) );
}
- CollectionInfo collection = getDefaultSchema().getCollection( headEntity.getType(), collName );
+ CollectionInfo collection = getDefaultSchema().getCollection( headEntity.getType(), collectionName );
if ( ( collection != null ) && !collection.getType().equals( itemType ) ) {
return null;
}
@@ -470,11 +467,11 @@ public class CpRelationManager implements RelationManager {
if ( itemEntity != null ) {
- addToCollection( collName, itemEntity );
+ addToCollection( collectionName, itemEntity );
if ( collection != null && collection.getLinkedCollection() != null ) {
Id itemEntityId = new SimpleId( itemEntity.getUuid(), itemEntity.getType() );
- final Edge edge = createCollectionEdge( cpHeadEntity.getId(), collName, itemEntityId );
+ final Edge edge = createCollectionEdge( cpHeadEntity.getId(), collectionName, itemEntityId );
GraphManager gm = managerCache.getGraphManager( applicationScope );
gm.writeEdge( edge );
@@ -486,11 +483,11 @@ public class CpRelationManager implements RelationManager {
@Override
- public void removeFromCollection( String collName, EntityRef itemRef ) throws Exception {
+ public void removeFromCollection( String collectionName, EntityRef itemRef ) throws Exception {
// special handling for roles collection of the application
if ( headEntity.getUuid().equals( applicationId ) ) {
- if ( collName.equals( COLLECTION_ROLES ) ) {
+ if ( collectionName.equals( COLLECTION_ROLES ) ) {
Entity itemEntity = em.get( itemRef );
if ( itemEntity != null ) {
RoleRef roleRef = SimpleRoleRef.forRoleEntity( itemEntity );
@@ -522,7 +519,7 @@ public class CpRelationManager implements RelationManager {
//run our delete
- final Edge collectionToItemEdge = createCollectionEdge( cpHeadEntity.getId(), collName, memberEntity.getId() );
+ final Edge collectionToItemEdge = createCollectionEdge( cpHeadEntity.getId(), collectionName, memberEntity.getId() );
gm.markEdge( collectionToItemEdge ).toBlocking().last();
@@ -535,7 +532,7 @@ public class CpRelationManager implements RelationManager {
final EntityIndexBatch batch = ei.createBatch();
// remove item from collection index
- SearchEdge indexScope = createCollectionSearchEdge( cpHeadEntity.getId(), collName );
+ SearchEdge indexScope = createCollectionSearchEdge( cpHeadEntity.getId(), collectionName );
batch.deindex( indexScope, memberEntity );
@@ -546,7 +543,7 @@ public class CpRelationManager implements RelationManager {
// special handling for roles collection of a group
if ( headEntity.getType().equals( Group.ENTITY_TYPE ) ) {
- if ( collName.equals( COLLECTION_ROLES ) ) {
+ if ( collectionName.equals( COLLECTION_ROLES ) ) {
String path = ( String ) ( ( Entity ) itemRef ).getMetadata( "path" );
if ( path.startsWith( "/roles/" ) ) {
@@ -599,20 +596,20 @@ public class CpRelationManager implements RelationManager {
@Override
- public Results searchCollection( String collName, Query query ) throws Exception {
+ public Results searchCollection( String collectionName, Query query ) throws Exception {
if ( query == null ) {
query = new Query();
- query.setCollection( collName );
+ query.setCollection( collectionName );
}
headEntity = em.validate( headEntity );
- CollectionInfo collection = getDefaultSchema().getCollection( headEntity.getType(), collName );
+ CollectionInfo collection = getDefaultSchema().getCollection( headEntity.getType(), collectionName );
if ( collection == null ) {
throw new RuntimeException(
- "Cannot find collection-info for '" + collName + "' of " + headEntity.getType() + ":" + headEntity
+ "Cannot find collection-info for '" + collectionName + "' of " + headEntity.getType() + ":" + headEntity
.getUuid() );
}
@@ -627,18 +624,18 @@ public class CpRelationManager implements RelationManager {
//set our fields applicable to both operations
readPipelineBuilder.withCursor(query.getCursor());
- readPipelineBuilder.withLimit( Optional.of( query.getLimit() ));
+ readPipelineBuilder.withLimit( Optional.of(query.getLimit()));
//TODO, this should be removed when the CP relation manager is removed
readPipelineBuilder.setStartId( cpHeadEntity.getId() );
if ( query.isGraphSearch() ) {
- readPipelineBuilder.getCollection( collName );
+ readPipelineBuilder.getCollection( collectionName );
}
else {
final String entityType = collection.getType();
- readPipelineBuilder.getCollectionWithQuery( collName, entityType, query.getQl().get() );
+ readPipelineBuilder.getCollectionWithQuery( collectionName, entityType, query.getQl().get() );
}
@@ -648,14 +645,14 @@ public class CpRelationManager implements RelationManager {
}
@Override
- public Results searchCollectionConsistent( String collName, Query query, int expectedResults ) throws Exception {
+ public Results searchCollectionConsistent( String collectionName, Query query, int expectedResults ) throws Exception {
Results results;
long maxLength = entityManagerFig.pollForRecordsTimeout();
long sleepTime = entityManagerFig.sleep();
boolean found;
long current = System.currentTimeMillis(), length = 0;
do {
- results = searchCollection(collName, query);
+ results = searchCollection(collectionName, query);
length = System.currentTimeMillis() - current;
found = expectedResults == results.size();
if(found){
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e3ba2b0b/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
index ebc1008..7909439 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
@@ -540,8 +540,7 @@ public class ManagementServiceImpl implements ManagementService {
organizationEntity = em.create( organizationEntity );
em.addToCollection( organizationEntity, "users", new SimpleEntityRef( User.ENTITY_TYPE, user.getUuid() ) );
- // em.addToCollection( new SimpleEntityRef( User.ENTITY_TYPE, user.getUuid() ), Schema.COLLECTION_GROUPS, organizationEntity );
-
+// em.addToCollection( new SimpleEntityRef( User.ENTITY_TYPE, user.getUuid() ), Schema.COLLECTION_GROUPS, organizationEntity );
writeUserToken( smf.getManagementAppId(), organizationEntity, encryptionService
.plainTextCredentials( generateOAuthSecretKey( AuthPrincipalType.ORGANIZATION ), user.getUuid(),
[04/22] incubator-usergrid git commit: Refactored observable methods
to correct name
Posted by gr...@apache.org.
Refactored observable methods to correct name
Added timestamp to message so that consumers can filter values for faster processing
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/cb179d35
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/cb179d35
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/cb179d35
Branch: refs/heads/USERGRID-608
Commit: cb179d3512952203d20c00773789df23e27f147d
Parents: b1d9ac2
Author: Todd Nine <tn...@apigee.com>
Authored: Tue May 12 17:40:20 2015 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Tue May 12 17:40:20 2015 -0700
----------------------------------------------------------------------
.../asyncevents/EventBuilder.java | 6 +-
.../asyncevents/EventBuilderImpl.java | 18 ++-
.../asyncevents/InMemoryAsyncEventService.java | 5 +-
.../asyncevents/SQSAsyncEventService.java | 9 +-
.../index/EntityIndexOperation.java | 46 +++++++
.../index/IndexServiceRequestBuilder.java | 88 +++++++++++++
.../index/IndexServiceRequestBuilderImpl.java | 130 +++++++++++++++++++
.../corepersistence/index/ReIndexAction.java | 2 +-
.../corepersistence/index/ReIndexService.java | 13 +-
.../index/ReIndexServiceImpl.java | 44 ++++---
.../rx/impl/AllEntityIdsObservable.java | 3 +-
.../rx/impl/AllEntityIdsObservableImpl.java | 5 +-
.../util/SerializableMapper.java | 91 -------------
.../rx/EdgesToTargetObservableIT.java | 4 +-
.../graph/serialization/EdgesObservable.java | 7 +-
.../serialization/impl/EdgesObservableImpl.java | 8 +-
.../impl/TargetIdObservableImpl.java | 2 +-
.../impl/migration/EdgeDataMigrationImpl.java | 2 +-
18 files changed, 342 insertions(+), 141 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cb179d35/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilder.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilder.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilder.java
index f48451c..f9f157e 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilder.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilder.java
@@ -22,8 +22,8 @@ package org.apache.usergrid.corepersistence.asyncevents;
import java.util.List;
+import org.apache.usergrid.corepersistence.index.EntityIndexOperation;
import org.apache.usergrid.persistence.collection.MvccLogEntry;
-import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.graph.Edge;
import org.apache.usergrid.persistence.index.impl.IndexOperationMessage;
@@ -72,10 +72,10 @@ public interface EventBuilder {
/**
* Re-index an entity in the scope provided
- * @param entityIdScope
+ * @param entityIndexOperation
* @return
*/
- Observable<IndexOperationMessage> index( EntityIdScope entityIdScope );
+ Observable<IndexOperationMessage> index( EntityIndexOperation entityIndexOperation );
/**
* A bean to hold both our observables so the caller can choose the subscription mechanism. Note that
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cb179d35/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
index c0d82d2..d35ed6d 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
@@ -25,12 +25,13 @@ import java.util.List;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.usergrid.corepersistence.index.EntityIndexOperation;
import org.apache.usergrid.corepersistence.index.IndexService;
+import org.apache.usergrid.persistence.Schema;
import org.apache.usergrid.persistence.collection.EntityCollectionManager;
import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
import org.apache.usergrid.persistence.collection.MvccLogEntry;
import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
-import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.graph.Edge;
import org.apache.usergrid.persistence.graph.GraphManager;
@@ -38,6 +39,7 @@ import org.apache.usergrid.persistence.graph.GraphManagerFactory;
import org.apache.usergrid.persistence.index.impl.IndexOperationMessage;
import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
+import org.apache.usergrid.persistence.model.field.Field;
import com.google.inject.Inject;
import com.google.inject.Singleton;
@@ -140,14 +142,20 @@ public class EventBuilderImpl implements EventBuilder {
@Override
- public Observable<IndexOperationMessage> index( final EntityIdScope entityIdScope ) {
+ public Observable<IndexOperationMessage> index( final EntityIndexOperation entityIndexOperation ) {
- final ApplicationScope applicationScope = entityIdScope.getApplicationScope();
+ final ApplicationScope applicationScope = entityIndexOperation.getApplicationScope();
- final Id entityId = entityIdScope.getId();
+ final Id entityId = entityIndexOperation.getId();
//load the entity
- return entityCollectionManagerFactory.createCollectionManager( applicationScope ).load( entityId )
+ return entityCollectionManagerFactory.createCollectionManager( applicationScope ).load( entityId ).filter(
+ entity -> {
+ final Field<Long> modified = entity.getField( Schema.PROPERTY_MODIFIED );
+
+ //only re-index if it has been updated and been updated after our timestamp
+ return modified != null && modified.getValue() >= entityIndexOperation.getUpdatedSince();
+ } )
//perform indexing on the task scheduler and start it
.flatMap( entity -> indexService.indexEntity( applicationScope, entity ) );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cb179d35/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/InMemoryAsyncEventService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/InMemoryAsyncEventService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/InMemoryAsyncEventService.java
index 6faa695..96966bf 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/InMemoryAsyncEventService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/InMemoryAsyncEventService.java
@@ -23,6 +23,7 @@ package org.apache.usergrid.corepersistence.asyncevents;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.usergrid.corepersistence.index.EntityIndexOperation;
import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
import org.apache.usergrid.persistence.core.rx.RxTaskScheduler;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
@@ -97,10 +98,10 @@ public class InMemoryAsyncEventService implements AsyncEventService {
@Override
- public void index( final EntityIdScope entityIdScope ) {
+ public void index( final EntityIndexOperation entityIndexOperation ) {
- run(eventBuilder.index( entityIdScope ));
+ run(eventBuilder.index( entityIndexOperation ));
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cb179d35/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/SQSAsyncEventService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/SQSAsyncEventService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/SQSAsyncEventService.java
index 415e5e8..1dbfd4e 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/SQSAsyncEventService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/SQSAsyncEventService.java
@@ -28,6 +28,7 @@ import java.util.concurrent.atomic.AtomicLong;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.usergrid.corepersistence.index.EntityIndexOperation;
import org.apache.usergrid.corepersistence.index.IndexProcessorFig;
import org.apache.usergrid.corepersistence.index.IndexService;
import org.apache.usergrid.exception.NotImplementedException;
@@ -184,7 +185,7 @@ public class SQSAsyncEventService implements AsyncEventService {
}
- @Override
+// @Override
public void index( final EntityIdScope entityIdScope ) {
//queue the re-inex operation
offer( entityIdScope );
@@ -346,4 +347,10 @@ public class SQSAsyncEventService implements AsyncEventService {
subscriptions.add( subscription );
}
}
+
+
+ @Override
+ public void index( final EntityIndexOperation entityIdScope ) {
+ throw new NotImplementedException( "Implement me" );
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cb179d35/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/EntityIndexOperation.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/EntityIndexOperation.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/EntityIndexOperation.java
new file mode 100644
index 0000000..3548bbe
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/EntityIndexOperation.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.index;
+
+
+import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+
+/**
+ * The operation for re-indexing an entity. The entity should be updated
+ * with an updated timestamp > updatedSince.
+ */
+public class EntityIndexOperation extends EntityIdScope {
+
+ private final long updatedSince;
+
+
+ public EntityIndexOperation( final ApplicationScope applicationScope, final Id id, final long updatedSince ) {
+ super( applicationScope, id );
+ this.updatedSince = updatedSince;
+ }
+
+
+ public long getUpdatedSince() {
+ return updatedSince;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cb179d35/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceRequestBuilder.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceRequestBuilder.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceRequestBuilder.java
new file mode 100644
index 0000000..07160d8
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceRequestBuilder.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.index;
+
+
+import java.util.UUID;
+
+import org.elasticsearch.action.index.IndexRequestBuilder;
+
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+
+import com.google.common.base.Optional;
+
+
+/**
+ * A builder interface to build our re-index request
+ */
+public interface IndexServiceRequestBuilder {
+
+ /**
+ * Set the application id
+ */
+ IndexServiceRequestBuilder withApplicationId( final UUID applicationId );
+
+ /**
+ * Set the collection name. If not set, every collection will be reindexed
+ * @param collectionName
+ * @return
+ */
+ IndexServiceRequestBuilder withCollection( final String collectionName );
+
+ /**
+ * Set our cursor to resume processing
+ * @param cursor
+ * @return
+ */
+ IndexServiceRequestBuilder withCursor(final String cursor);
+
+
+ /**
+ * Set the timestamp to re-index entities updated >= this timestamp
+ * @param timestamp
+ * @return
+ */
+ IndexServiceRequestBuilder withStartTimestamp(final Long timestamp);
+
+
+ /**
+ * Get the application scope
+ * @return
+ */
+ Optional<ApplicationScope> getApplicationScope();
+
+ /**
+ * Get the collection name
+ * @return
+ */
+ Optional<String> getCollectionName();
+
+ /**
+ * Get the cursor
+ * @return
+ */
+ Optional<String> getCursor();
+
+ /**
+ * Get the updated since timestamp
+ * @return
+ */
+ Optional<Long> getUpdateTimestamp();
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cb179d35/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceRequestBuilderImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceRequestBuilderImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceRequestBuilderImpl.java
new file mode 100644
index 0000000..3466674
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceRequestBuilderImpl.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.index;
+
+
+import java.util.UUID;
+
+
+import org.apache.usergrid.corepersistence.util.CpNamingUtils;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+
+import com.google.common.base.Optional;
+
+
+public class IndexServiceRequestBuilderImpl implements IndexServiceRequestBuilder {
+
+
+ /**
+ *
+ final Observable<ApplicationScope> applicationScopes = appId.isPresent()? Observable.just( getApplicationScope(appId.get()) ) : allApplicationsObservable.getData();
+
+ final String newCursor = StringUtils.sanitizeUUID( UUIDGenerator.newTimeUUID() );
+
+ //create an observable that loads each entity and indexes it, start it running with publish
+ final ConnectableObservable<EdgeScope> runningReIndex =
+ allEntityIdsObservable.getEdgesToEntities( applicationScopes, collection, startTimestamp )
+
+ //for each edge, create our scope and index on it
+ .doOnNext( edge -> indexService.index( new EntityIdScope( edge.getApplicationScope(), edge.getEdge().getTargetNode() ) ) ).publish();
+
+
+
+ //start our sampler and state persistence
+ //take a sample every sample interval to allow us to resume state with minimal loss
+ runningReIndex.sample( indexProcessorFig.getReIndexSampleInterval(), TimeUnit.MILLISECONDS,
+ rxTaskScheduler.getAsyncIOScheduler() )
+ .doOnNext( edge -> {
+
+ final String serializedState = SerializableMapper.asString( edge );
+
+ mapManager.putString( newCursor, serializedState, INDEX_TTL );
+ } ).subscribe();
+
+
+ */
+
+ private Optional<UUID> withApplicationId;
+ private Optional<String> withCollectionName;
+ private Optional<String> cursor;
+ private Optional<Long> updateTimestamp;
+
+
+ /***
+ *
+ * @param applicationId
+ * @return
+ */
+ @Override
+ public IndexServiceRequestBuilder withApplicationId( final UUID applicationId ) {
+ this.withApplicationId = Optional.fromNullable(applicationId);
+ return this;
+ }
+
+
+ @Override
+ public IndexServiceRequestBuilder withCollection( final String collectionName ) {
+ this.withCollectionName = Optional.fromNullable( collectionName );
+ return this;
+ }
+
+
+ @Override
+ public IndexServiceRequestBuilder withCursor( final String cursor ) {
+ this.cursor = Optional.fromNullable( cursor );
+ return this;
+ }
+
+
+ @Override
+ public IndexServiceRequestBuilder withStartTimestamp( final Long timestamp ) {
+ this.updateTimestamp = Optional.fromNullable(timestamp );
+ return this;
+ }
+
+
+ @Override
+ public Optional<ApplicationScope> getApplicationScope() {
+
+ if(this.withApplicationId.isPresent()){
+ return Optional.of( CpNamingUtils.getApplicationScope( withApplicationId.get()));
+ }
+
+ return Optional.absent();
+ }
+
+
+ @Override
+ public Optional<String> getCollectionName() {
+ return withCollectionName;
+ }
+
+
+ @Override
+ public Optional<String> getCursor() {
+ return cursor;
+ }
+
+
+ @Override
+ public Optional<Long> getUpdateTimestamp() {
+ return updateTimestamp;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cb179d35/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexAction.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexAction.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexAction.java
index b878246..672b3c8 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexAction.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexAction.java
@@ -33,5 +33,5 @@ public interface ReIndexAction {
* Index this entity with the specified scope
* @param entityIdScope
*/
- void index( final EntityIdScope entityIdScope );
+ void index( final EntityIndexOperation entityIdScope );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cb179d35/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java
index e594ad3..b25eca5 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java
@@ -45,16 +45,17 @@ public interface ReIndexService {
/**
* Perform an index rebuild
*
- * @param appId The applicationId to re-index, or all applications if absent
- * @param collection The collection name to re-index. Otherwise all collections in an app will be used.
- * @param cursor An optional cursor to resume processing
- * @param startTimestamp The time to start indexing from. All edges >= this time will be indexed.
+ * @param indexServiceRequestBuilder The builder to build the request
* @return
*/
- IndexResponse rebuildIndex( final Optional<UUID> appId, final Optional<String> collection, final Optional<String> cursor,
- final Optional<Long> startTimestamp);
+ IndexResponse rebuildIndex(final IndexServiceRequestBuilder indexServiceRequestBuilder);
+ /**
+ * Generate a build for the index
+ * @return
+ */
+ IndexServiceRequestBuilder getBuilder();
/**
* The response when requesting a re-index operation
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cb179d35/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
index bd1bff9..a2fa09a 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
@@ -20,7 +20,6 @@
package org.apache.usergrid.corepersistence.index;
-import java.util.UUID;
import java.util.concurrent.TimeUnit;
import org.apache.usergrid.corepersistence.asyncevents.AsyncEventService;
@@ -28,7 +27,6 @@ import org.apache.usergrid.corepersistence.rx.impl.AllApplicationsObservable;
import org.apache.usergrid.corepersistence.rx.impl.AllEntityIdsObservable;
import org.apache.usergrid.corepersistence.rx.impl.EdgeScope;
import org.apache.usergrid.corepersistence.util.CpNamingUtils;
-import org.apache.usergrid.corepersistence.util.SerializableMapper;
import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
import org.apache.usergrid.persistence.core.rx.RxTaskScheduler;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
@@ -46,13 +44,11 @@ import com.google.inject.Singleton;
import rx.Observable;
import rx.observables.ConnectableObservable;
-import static org.apache.usergrid.corepersistence.util.CpNamingUtils.getApplicationScope;
-
@Singleton
public class ReIndexServiceImpl implements ReIndexService {
- private static final MapScope RESUME_MAP_SCOPTE =
+ private static final MapScope RESUME_MAP_SCOPE =
new MapScopeImpl( CpNamingUtils.getManagementApplicationId(), "reindexresume" );
//Keep cursors to resume re-index for 1 day. This is far beyond it's useful real world implications anyway.
@@ -78,31 +74,41 @@ public class ReIndexServiceImpl implements ReIndexService {
this.rxTaskScheduler = rxTaskScheduler;
this.indexService = indexService;
- this.mapManager = mapManagerFactory.createMapManager( RESUME_MAP_SCOPTE );
+ this.mapManager = mapManagerFactory.createMapManager( RESUME_MAP_SCOPE );
}
+
+
@Override
- public IndexResponse rebuildIndex( final Optional<UUID> appId, final Optional<String> collection, final Optional<String> cursor,
- final Optional<Long> startTimestamp ) {
+ public IndexResponse rebuildIndex( final IndexServiceRequestBuilder indexServiceRequestBuilder ) {
- //load our last emitted Scope if a cursor is present
- if ( cursor.isPresent() ) {
+ //load our last emitted Scope if a cursor is present
+ if ( indexServiceRequestBuilder.getCursor().isPresent() ) {
throw new UnsupportedOperationException( "Build this" );
}
- final Observable<ApplicationScope> applicationScopes = appId.isPresent()? Observable.just( getApplicationScope(appId.get()) ) : allApplicationsObservable.getData();
+ final Optional<ApplicationScope> appId = indexServiceRequestBuilder.getApplicationScope();
+ final Observable<ApplicationScope> applicationScopes = appId.isPresent()? Observable.just( appId.get() ) : allApplicationsObservable.getData();
+
+
+
final String newCursor = StringUtils.sanitizeUUID( UUIDGenerator.newTimeUUID() );
+ final long modifiedSince = indexServiceRequestBuilder.getUpdateTimestamp().or( Long.MIN_VALUE );
+
//create an observable that loads each entity and indexes it, start it running with publish
final ConnectableObservable<EdgeScope> runningReIndex =
- allEntityIdsObservable.getEdgesToEntities( applicationScopes, collection, startTimestamp )
+ allEntityIdsObservable.getEdgesToEntities( applicationScopes,
+ indexServiceRequestBuilder.getCollectionName() )
//for each edge, create our scope and index on it
- .doOnNext( edge -> indexService.index( new EntityIdScope( edge.getApplicationScope(), edge.getEdge().getTargetNode() ) ) ).publish();
+ .doOnNext( edge -> indexService.index(
+ new EntityIndexOperation( edge.getApplicationScope(), edge.getEdge().getTargetNode(),
+ modifiedSince ) ) ).publish();
@@ -112,9 +118,9 @@ public class ReIndexServiceImpl implements ReIndexService {
rxTaskScheduler.getAsyncIOScheduler() )
.doOnNext( edge -> {
- final String serializedState = SerializableMapper.asString( edge );
-
- mapManager.putString( newCursor, serializedState, INDEX_TTL );
+// final String serializedState = SerializableMapper.asString( edge );
+//
+// mapManager.putString( newCursor, serializedState, INDEX_TTL );
} ).subscribe();
@@ -124,6 +130,12 @@ public class ReIndexServiceImpl implements ReIndexService {
return new IndexResponse( newCursor, runningReIndex );
}
+
+
+ @Override
+ public IndexServiceRequestBuilder getBuilder() {
+ return new IndexServiceRequestBuilderImpl();
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cb179d35/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntityIdsObservable.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntityIdsObservable.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntityIdsObservable.java
index b9e5373..aada240 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntityIdsObservable.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntityIdsObservable.java
@@ -44,9 +44,8 @@ public interface AllEntityIdsObservable {
* Get all edges that represent edges to entities in the system
* @param appScopes
* @param edgeType The edge type to use (if specified)
- * @param startTime The time to start with
* @return
*/
- Observable<EdgeScope> getEdgesToEntities(final Observable<ApplicationScope> appScopes, final Optional<String> edgeType, final Optional<Long> startTime);
+ Observable<EdgeScope> getEdgesToEntities(final Observable<ApplicationScope> appScopes, final Optional<String> edgeType);
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cb179d35/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntityIdsObservableImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntityIdsObservableImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntityIdsObservableImpl.java
index 257fab1..6a95e7b 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntityIdsObservableImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntityIdsObservableImpl.java
@@ -81,12 +81,13 @@ public class AllEntityIdsObservableImpl implements AllEntityIdsObservable {
@Override
- public Observable<EdgeScope> getEdgesToEntities( final Observable<ApplicationScope> appScopes, final Optional<String> edgeType, final Optional<Long> startTime) {
+ public Observable<EdgeScope> getEdgesToEntities( final Observable<ApplicationScope> appScopes, final Optional<String> edgeType) {
return appScopes.flatMap( applicationScope -> {
final GraphManager gm = graphManagerFactory.createEdgeManager( applicationScope );
- return edgesObservable.edgesFromSourceAscending( gm, applicationScope.getApplication(),edgeType, startTime ).map( edge -> new EdgeScope(applicationScope, edge ));
+ return edgesObservable.edgesFromSourceDescending( gm, applicationScope.getApplication(), edgeType )
+ .map( edge -> new EdgeScope(applicationScope, edge ));
} );
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cb179d35/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/SerializableMapper.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/SerializableMapper.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/SerializableMapper.java
deleted file mode 100644
index 19ecf6d..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/SerializableMapper.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.util;
-
-
-import java.io.ByteArrayInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.Serializable;
-import java.nio.charset.StandardCharsets;
-
-import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
-import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
-
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.dataformat.smile.SmileFactory;
-import com.google.common.base.Preconditions;
-
-
-/**
- * A simple utility for serializing serializable classes to/and from strings. To be used for small object storage only, such as resume on re-index
- * storing data such as entities should be specialized.
- */
-public class SerializableMapper {
-
- private static final SmileFactory SMILE_FACTORY = new SmileFactory();
-
- private static final ObjectMapper MAPPER = new ObjectMapper( SMILE_FACTORY );
-
- static{
- MAPPER.enableDefaultTypingAsProperty( ObjectMapper.DefaultTyping.JAVA_LANG_OBJECT, "@class" );
- SMILE_FACTORY.delegateToTextual( true );
- }
-
- /**
- * Get value as a string
- * @param toSerialize
- * @param <T>
- * @return
- */
- public static <T extends Serializable> String asString(final T toSerialize){
- try {
- return MAPPER.writeValueAsString( toSerialize );
- }
- catch ( JsonProcessingException e ) {
- throw new RuntimeException( "Unable to process json", e );
- }
- }
-
-
- /**
- * Write the value as a string
- * @param <T>
- * @param serialized
- * @param clazz
- * @return
- */
- public static <T extends Serializable> T fromString(final String serialized, final Class<T> clazz){
- Preconditions.checkNotNull(serialized, "serialized string cannot be null");
-
-
- InputStream stream = new ByteArrayInputStream(serialized.getBytes( StandardCharsets.UTF_8));
-
- try {
- return MAPPER.readValue( stream, clazz );
- }
- catch ( IOException e ) {
- throw new RuntimeException( String.format("Unable to parse string '%s'", serialized), e );
- }
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cb179d35/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/EdgesToTargetObservableIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/EdgesToTargetObservableIT.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/EdgesToTargetObservableIT.java
index 92f2b01..9e84219 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/EdgesToTargetObservableIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/EdgesToTargetObservableIT.java
@@ -89,7 +89,7 @@ public class EdgesToTargetObservableIT extends AbstractCoreIT {
final GraphManager gm = managerCache.getGraphManager( scope );
- edgesFromSourceObservable.edgesFromSourceAscending( gm, applicationId ).doOnNext( edge -> {
+ edgesFromSourceObservable.edgesFromSourceDescending( gm, applicationId ).doOnNext( edge -> {
final String edgeType = edge.getType();
final Id target = edge.getTargetNode();
@@ -118,7 +118,7 @@ public class EdgesToTargetObservableIT extends AbstractCoreIT {
//test connections
- edgesFromSourceObservable.edgesFromSourceAscending( gm, source ).doOnNext( edge -> {
+ edgesFromSourceObservable.edgesFromSourceDescending( gm, source ).doOnNext( edge -> {
final String edgeType = edge.getType();
final Id target = edge.getTargetNode();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cb179d35/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/EdgesObservable.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/EdgesObservable.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/EdgesObservable.java
index 9f0bd60..964e13d 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/EdgesObservable.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/EdgesObservable.java
@@ -38,7 +38,7 @@ public interface EdgesObservable {
* @param sourceNode
* @return
*/
- Observable<Edge> edgesFromSourceAscending( final GraphManager gm, final Id sourceNode );
+ Observable<Edge> edgesFromSourceDescending( final GraphManager gm, final Id sourceNode );
/**
@@ -46,11 +46,10 @@ public interface EdgesObservable {
* @param gm
* @param sourceNode
* @param edgeType The edge type if specified. Otherwise all types will be used
- * @param startTimestamp The start timestamp if specfiied, otherwise Long.MIN will be used
* @return
*/
- Observable<Edge> edgesFromSourceAscending( final GraphManager gm, final Id sourceNode,final Optional<String> edgeType,
- final Optional<Long> startTimestamp );
+ Observable<Edge> edgesFromSourceDescending( final GraphManager gm, final Id sourceNode,
+ final Optional<String> edgeType );
/**
* Get all edges from the source node with the target type
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cb179d35/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgesObservableImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgesObservableImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgesObservableImpl.java
index df9e094..7240798 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgesObservableImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgesObservableImpl.java
@@ -55,7 +55,7 @@ public class EdgesObservableImpl implements EdgesObservable {
* Get all edges from the source
*/
@Override
- public Observable<Edge> edgesFromSourceAscending( final GraphManager gm, final Id sourceNode ) {
+ public Observable<Edge> edgesFromSourceDescending( final GraphManager gm, final Id sourceNode ) {
final Observable<String> edgeTypes =
gm.getEdgeTypesFromSource( new SimpleSearchEdgeType( sourceNode, null, null ) );
@@ -71,8 +71,8 @@ public class EdgesObservableImpl implements EdgesObservable {
@Override
- public Observable<Edge> edgesFromSourceAscending( final GraphManager gm, final Id sourceNode, final Optional<String> edgeTypeInput,
- final Optional<Long> startTimestamp ) {
+ public Observable<Edge> edgesFromSourceDescending( final GraphManager gm, final Id sourceNode,
+ final Optional<String> edgeTypeInput ) {
@@ -85,7 +85,7 @@ public class EdgesObservableImpl implements EdgesObservable {
logger.debug( "Loading edges of edgeType {} from {}", edgeType, sourceNode );
return gm.loadEdgesFromSource(
- new SimpleSearchByEdgeType( sourceNode, edgeType, startTimestamp.or( Long.MIN_VALUE ), SearchByEdgeType.Order.ASCENDING,
+ new SimpleSearchByEdgeType( sourceNode, edgeType, Long.MAX_VALUE, SearchByEdgeType.Order.DESCENDING,
Optional.<Edge>absent() ) );
} );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cb179d35/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/TargetIdObservableImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/TargetIdObservableImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/TargetIdObservableImpl.java
index 5cf5117..82c7d54 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/TargetIdObservableImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/TargetIdObservableImpl.java
@@ -55,7 +55,7 @@ public class TargetIdObservableImpl implements TargetIdObservable {
public Observable<Id> getTargetNodes(final GraphManager gm, final Id sourceNode) {
//only search edge types that start with collections
- return edgesFromSourceObservable.edgesFromSourceAscending( gm, sourceNode ).map( new Func1<Edge, Id>() {
+ return edgesFromSourceObservable.edgesFromSourceDescending( gm, sourceNode ).map( new Func1<Edge, Id>() {
@Override
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cb179d35/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/EdgeDataMigrationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/EdgeDataMigrationImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/EdgeDataMigrationImpl.java
index 0df26ff..d6c42e3 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/EdgeDataMigrationImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/EdgeDataMigrationImpl.java
@@ -87,7 +87,7 @@ public class EdgeDataMigrationImpl implements DataMigration<GraphNode> {
final GraphManager gm = graphManagerFactory.createEdgeManager( graphNode.applicationScope );
//get edges from the source
- return edgesFromSourceObservable.edgesFromSourceAscending( gm, graphNode.entryNode ).buffer( 1000 )
+ return edgesFromSourceObservable.edgesFromSourceDescending( gm, graphNode.entryNode ).buffer( 1000 )
.doOnNext( edges -> {
final MutationBatch batch = keyspace.prepareMutationBatch();
[14/22] incubator-usergrid git commit: fix serviceinvocation it
Posted by gr...@apache.org.
fix serviceinvocation it
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/b8a95e92
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/b8a95e92
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/b8a95e92
Branch: refs/heads/USERGRID-608
Commit: b8a95e92e683d3ce6bdc41a486a655791cfd5d52
Parents: c7a4b19
Author: Shawn Feldman <sf...@apache.org>
Authored: Fri May 15 13:23:16 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Fri May 15 13:23:16 2015 -0600
----------------------------------------------------------------------
.../org/apache/usergrid/corepersistence/CpRelationManager.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b8a95e92/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index 6690346..7f3b62b 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -260,7 +260,7 @@ public class CpRelationManager implements RelationManager {
GraphManager gm = managerCache.getGraphManager( applicationScope );
Observable<Edge> edges = gm.loadEdgeVersions(
new SimpleSearchByEdge( new SimpleId( headEntity.getUuid(), headEntity.getType() ), edgeType, entityId,
- Long.MAX_VALUE, SearchByEdgeType.Order.DESCENDING, null ) );
+ Long.MAX_VALUE, SearchByEdgeType.Order.DESCENDING, Optional.absent() ) );
return edges.toBlocking().firstOrDefault( null ) != null;
}
[11/22] incubator-usergrid git commit: Fixes bugs and cleans up tests
Posted by gr...@apache.org.
Fixes bugs and cleans up tests
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/48be894f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/48be894f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/48be894f
Branch: refs/heads/USERGRID-608
Commit: 48be894fb8af857f33700fcc2717357936d12913
Parents: a767bb6
Author: Todd Nine <tn...@apigee.com>
Authored: Thu May 14 19:23:04 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Thu May 14 19:23:04 2015 -0600
----------------------------------------------------------------------
.../usergrid/corepersistence/CoreModule.java | 4 +
.../corepersistence/CpRelationManager.java | 1 -
.../index/IndexProcessorFig.java | 2 +-
.../index/IndexServiceRequestBuilderImpl.java | 74 ++---
.../corepersistence/index/ReIndexService.java | 16 +-
.../index/ReIndexServiceImpl.java | 48 ++-
.../cursor/AbstractCursorSerializer.java | 2 +-
.../PerformanceEntityRebuildIndexTest.java | 318 +++++++++----------
8 files changed, 225 insertions(+), 240 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/48be894f/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
index a02bffd..b22a7cb 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
@@ -25,6 +25,8 @@ import org.apache.usergrid.corepersistence.asyncevents.EventBuilderImpl;
import org.apache.usergrid.corepersistence.index.IndexProcessorFig;
import org.apache.usergrid.corepersistence.index.IndexService;
import org.apache.usergrid.corepersistence.index.IndexServiceImpl;
+import org.apache.usergrid.corepersistence.index.ReIndexService;
+import org.apache.usergrid.corepersistence.index.ReIndexServiceImpl;
import org.apache.usergrid.corepersistence.migration.AppInfoMigrationPlugin;
import org.apache.usergrid.corepersistence.migration.CoreMigration;
import org.apache.usergrid.corepersistence.migration.CoreMigrationPlugin;
@@ -142,6 +144,8 @@ public class CoreModule extends AbstractModule {
bind( AsyncEventService.class ).toProvider( AsyncIndexProvider.class );
+ bind( ReIndexService.class).to( ReIndexServiceImpl.class );
+
install( new GuicyFigModule( IndexProcessorFig.class ) );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/48be894f/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index 4993d88..cba1a07 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -661,7 +661,6 @@ public class CpRelationManager implements RelationManager {
}while (!found && length <= maxLength);
if(logger.isInfoEnabled()){
logger.info(String.format("Consistent Search finished in %s, results=%s, expected=%s...dumping stack",length, results.size(),expectedResults));
- Thread.dumpStack();
}
return results;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/48be894f/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java
index 8e835e2..e4b2329 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java
@@ -78,7 +78,7 @@ public interface IndexProcessorFig extends GuicyFig {
String getQueueImplementation();
- @Default("10000")
+ @Default("1000")
@Key("elasticsearch.reindex.flush.interval")
int getUpdateInterval();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/48be894f/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceRequestBuilderImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceRequestBuilderImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceRequestBuilderImpl.java
index 3466674..4017b6e 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceRequestBuilderImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceRequestBuilderImpl.java
@@ -22,70 +22,57 @@ package org.apache.usergrid.corepersistence.index;
import java.util.UUID;
-
import org.apache.usergrid.corepersistence.util.CpNamingUtils;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import com.google.common.base.Optional;
+/**
+ * Index service request builder
+ */
public class IndexServiceRequestBuilderImpl implements IndexServiceRequestBuilder {
-
- /**
- *
- final Observable<ApplicationScope> applicationScopes = appId.isPresent()? Observable.just( getApplicationScope(appId.get()) ) : allApplicationsObservable.getData();
-
- final String newCursor = StringUtils.sanitizeUUID( UUIDGenerator.newTimeUUID() );
-
- //create an observable that loads each entity and indexes it, start it running with publish
- final ConnectableObservable<EdgeScope> runningReIndex =
- allEntityIdsObservable.getEdgesToEntities( applicationScopes, collection, startTimestamp )
-
- //for each edge, create our scope and index on it
- .doOnNext( edge -> indexService.index( new EntityIdScope( edge.getApplicationScope(), edge.getEdge().getTargetNode() ) ) ).publish();
-
-
-
- //start our sampler and state persistence
- //take a sample every sample interval to allow us to resume state with minimal loss
- runningReIndex.sample( indexProcessorFig.getReIndexSampleInterval(), TimeUnit.MILLISECONDS,
- rxTaskScheduler.getAsyncIOScheduler() )
- .doOnNext( edge -> {
-
- final String serializedState = SerializableMapper.asString( edge );
-
- mapManager.putString( newCursor, serializedState, INDEX_TTL );
- } ).subscribe();
-
-
- */
-
- private Optional<UUID> withApplicationId;
- private Optional<String> withCollectionName;
- private Optional<String> cursor;
- private Optional<Long> updateTimestamp;
+ private Optional<UUID> withApplicationId = Optional.absent();
+ private Optional<String> withCollectionName = Optional.absent();
+ private Optional<String> cursor = Optional.absent();
+ private Optional<Long> updateTimestamp = Optional.absent();
/***
*
- * @param applicationId
+ * @param applicationId The application id
* @return
*/
@Override
public IndexServiceRequestBuilder withApplicationId( final UUID applicationId ) {
- this.withApplicationId = Optional.fromNullable(applicationId);
+ this.withApplicationId = Optional.fromNullable( applicationId );
return this;
}
+ /**
+ * the colleciton name
+ * @param collectionName
+ * @return
+ */
@Override
public IndexServiceRequestBuilder withCollection( final String collectionName ) {
- this.withCollectionName = Optional.fromNullable( collectionName );
+ if(collectionName == null){
+ this.withCollectionName = Optional.absent();
+ }
+ else {
+ this.withCollectionName = Optional.fromNullable( CpNamingUtils.getEdgeTypeFromCollectionName( collectionName ) );
+ }
return this;
}
+ /**
+ * The cursor
+ * @param cursor
+ * @return
+ */
@Override
public IndexServiceRequestBuilder withCursor( final String cursor ) {
this.cursor = Optional.fromNullable( cursor );
@@ -93,9 +80,14 @@ public class IndexServiceRequestBuilderImpl implements IndexServiceRequestBuilde
}
+ /**
+ * Set start timestamp in epoch time. Only entities updated since this time will be processed for indexing
+ * @param timestamp
+ * @return
+ */
@Override
public IndexServiceRequestBuilder withStartTimestamp( final Long timestamp ) {
- this.updateTimestamp = Optional.fromNullable(timestamp );
+ this.updateTimestamp = Optional.fromNullable( timestamp );
return this;
}
@@ -103,8 +95,8 @@ public class IndexServiceRequestBuilderImpl implements IndexServiceRequestBuilde
@Override
public Optional<ApplicationScope> getApplicationScope() {
- if(this.withApplicationId.isPresent()){
- return Optional.of( CpNamingUtils.getApplicationScope( withApplicationId.get()));
+ if ( this.withApplicationId.isPresent() ) {
+ return Optional.of( CpNamingUtils.getApplicationScope( withApplicationId.get() ) );
}
return Optional.absent();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/48be894f/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java
index f8955dd..af3615e 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java
@@ -31,7 +31,7 @@ public interface ReIndexService {
*
* @param indexServiceRequestBuilder The builder to build the request
*/
- IndexResponse rebuildIndex( final IndexServiceRequestBuilder indexServiceRequestBuilder );
+ ReIndexStatus rebuildIndex( final IndexServiceRequestBuilder indexServiceRequestBuilder );
/**
@@ -45,20 +45,20 @@ public interface ReIndexService {
* @param jobId The jobId returned during the rebuild index
* @return
*/
- IndexResponse getStatus( final String jobId );
+ ReIndexStatus getStatus( final String jobId );
/**
* The response when requesting a re-index operation
*/
- class IndexResponse {
+ class ReIndexStatus {
final String jobId;
- final String status;
+ final Status status;
final long numberProcessed;
final long lastUpdated;
- public IndexResponse( final String jobId, final String status, final long numberProcessed,
+ public ReIndexStatus( final String jobId, final Status status, final long numberProcessed,
final long lastUpdated ) {
this.jobId = jobId;
this.status = status;
@@ -97,8 +97,12 @@ public interface ReIndexService {
* Get the status
* @return
*/
- public String getStatus() {
+ public Status getStatus() {
return status;
}
}
+
+ enum Status{
+ STARTED, INPROGRESS, COMPLETE;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/48be894f/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
index d828fc2..f44113b 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
@@ -22,6 +22,10 @@ package org.apache.usergrid.corepersistence.index;
import java.util.List;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import org.apache.usergrid.corepersistence.asyncevents.AsyncEventService;
import org.apache.usergrid.corepersistence.pipeline.cursor.CursorSerializerUtil;
import org.apache.usergrid.corepersistence.pipeline.read.CursorSeek;
@@ -51,6 +55,8 @@ import rx.schedulers.Schedulers;
@Singleton
public class ReIndexServiceImpl implements ReIndexService {
+ private static final Logger logger = LoggerFactory.getLogger( ReIndexServiceImpl.class );
+
private static final MapScope RESUME_MAP_SCOPE =
new MapScopeImpl( CpNamingUtils.getManagementApplicationId(), "reindexresume" );
@@ -85,7 +91,7 @@ public class ReIndexServiceImpl implements ReIndexService {
@Override
- public IndexResponse rebuildIndex( final IndexServiceRequestBuilder indexServiceRequestBuilder ) {
+ public ReIndexStatus rebuildIndex( final IndexServiceRequestBuilder indexServiceRequestBuilder ) {
//load our last emitted Scope if a cursor is present
@@ -97,7 +103,7 @@ public class ReIndexServiceImpl implements ReIndexService {
final Optional<ApplicationScope> appId = indexServiceRequestBuilder.getApplicationScope();
- Preconditions.checkArgument( cursor.isPresent() && appId.isPresent(),
+ Preconditions.checkArgument( !(cursor.isPresent() && appId.isPresent()),
"You cannot specify an app id and a cursor. When resuming with cursor you must omit the appid" );
final Observable<ApplicationScope> applicationScopes = getApplications( cursor, appId );
@@ -112,9 +118,14 @@ public class ReIndexServiceImpl implements ReIndexService {
indexServiceRequestBuilder.getCollectionName(), cursorSeek.getSeekValue() )
//for each edge, create our scope and index on it
- .doOnNext( edge -> indexService.index(
- new EntityIndexOperation( edge.getApplicationScope(), edge.getEdge().getTargetNode(),
- modifiedSince ) ) );
+ .doOnNext( edge -> {
+ final EntityIndexOperation entityIndexOperation = new EntityIndexOperation( edge.getApplicationScope(), edge.getEdge().getTargetNode(), modifiedSince );
+
+ logger.info( "Queueing {}", entityIndexOperation );
+
+ indexService.index(entityIndexOperation);
+
+ } );
//start our sampler and state persistence
@@ -127,7 +138,7 @@ public class ReIndexServiceImpl implements ReIndexService {
.subscribeOn( Schedulers.io() ).subscribe();
- return new IndexResponse( jobId, "Started", 0, 0 );
+ return new ReIndexStatus( jobId, Status.STARTED, 0, 0 );
}
@@ -138,7 +149,7 @@ public class ReIndexServiceImpl implements ReIndexService {
@Override
- public IndexResponse getStatus( final String jobId ) {
+ public ReIndexStatus getStatus( final String jobId ) {
Preconditions.checkNotNull( jobId, "jobId must not be null" );
return getIndexResponse( jobId );
}
@@ -166,11 +177,11 @@ public class ReIndexServiceImpl implements ReIndexService {
writeCursorState( jobId, buffer.get( buffer.size() - 1 ) );
}
- writeStateMeta( jobId, "InProgress", count, System.currentTimeMillis() );
+ writeStateMeta( jobId, Status.INPROGRESS, count, System.currentTimeMillis() );
}
public void complete(){
- writeStateMeta( jobId, "Complete", count, System.currentTimeMillis() );
+ writeStateMeta( jobId, Status.COMPLETE, count, System.currentTimeMillis() );
}
}
@@ -257,10 +268,15 @@ public class ReIndexServiceImpl implements ReIndexService {
* @param processedCount
* @param lastUpdated
*/
- private void writeStateMeta( final String jobId, final String status, final long processedCount,
+ private void writeStateMeta( final String jobId, final Status status, final long processedCount,
final long lastUpdated ) {
- mapManager.putString( jobId + MAP_STATUS_KEY, status );
+ if(logger.isDebugEnabled()) {
+ logger.debug( "Flushing state for jobId {}, status {}, processedCount {}, lastUpdated {}",
+ new Object[] { jobId, status, processedCount, lastUpdated } );
+ }
+
+ mapManager.putString( jobId + MAP_STATUS_KEY, status.name() );
mapManager.putLong( jobId + MAP_COUNT_KEY, processedCount );
mapManager.putLong( jobId + MAP_UPDATED_KEY, lastUpdated );
}
@@ -271,18 +287,20 @@ public class ReIndexServiceImpl implements ReIndexService {
* @param jobId
* @return
*/
- private IndexResponse getIndexResponse( final String jobId ) {
+ private ReIndexStatus getIndexResponse( final String jobId ) {
- final String status = mapManager.getString( jobId+MAP_STATUS_KEY );
+ final String stringStatus = mapManager.getString( jobId+MAP_STATUS_KEY );
- if(status == null){
+ if(stringStatus == null){
throw new IllegalArgumentException( "Could not find a job with id " + jobId );
}
+ final Status status = Status.valueOf( stringStatus );
+
final long processedCount = mapManager.getLong( jobId + MAP_COUNT_KEY );
final long lastUpdated = mapManager.getLong( jobId + MAP_COUNT_KEY );
- return new IndexResponse( jobId, status, processedCount, lastUpdated );
+ return new ReIndexStatus( jobId, status, processedCount, lastUpdated );
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/48be894f/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/cursor/AbstractCursorSerializer.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/cursor/AbstractCursorSerializer.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/cursor/AbstractCursorSerializer.java
index 23bb99a..e770a77 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/cursor/AbstractCursorSerializer.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/cursor/AbstractCursorSerializer.java
@@ -44,7 +44,7 @@ public abstract class AbstractCursorSerializer<T> implements CursorSerializer<T>
try {
final Class<? extends T> classType = getType();
- return objectMapper.treeToValue( node, classType );
+ return objectMapper.treeToValue( node, classType );
}
catch ( JsonProcessingException e ) {
throw new CursorParseException( "Unable to deserialize value", e );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/48be894f/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java b/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
index 55c4846..8d54043 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
@@ -22,78 +22,67 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
-import java.util.concurrent.TimeUnit;
-import com.google.common.base.Optional;
-import org.apache.commons.lang.RandomStringUtils;
-
-import org.apache.usergrid.corepersistence.index.IndexServiceRequestBuilder;
-import org.apache.usergrid.corepersistence.index.ReIndexService;
-import org.apache.usergrid.persistence.index.ApplicationEntityIndex;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.commons.lang.RandomStringUtils;
+
import org.apache.usergrid.AbstractCoreIT;
import org.apache.usergrid.cassandra.SpringResource;
+import org.apache.usergrid.corepersistence.index.IndexServiceRequestBuilder;
+import org.apache.usergrid.corepersistence.index.ReIndexService;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
+import org.apache.usergrid.persistence.index.ApplicationEntityIndex;
import org.apache.usergrid.persistence.index.EntityIndexFactory;
import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.entity.SimpleId;
-import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
-import com.codahale.metrics.Slf4jReporter;
import com.google.inject.Injector;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.fail;
//@RunWith(JukitoRunner.class)
//@UseModules({ GuiceModule.class })
+
public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT {
- private static final Logger logger = LoggerFactory.getLogger(PerformanceEntityRebuildIndexTest.class );
+ private static final Logger logger = LoggerFactory.getLogger( PerformanceEntityRebuildIndexTest.class );
private static final MetricRegistry registry = new MetricRegistry();
- private Slf4jReporter reporter;
-
- private static final int ENTITIES_TO_INDEX = 2000;
+ private static final int ENTITIES_TO_INDEX = 1000;
@Before
public void startReporting() {
- logger.debug("Starting metrics reporting");
- reporter = Slf4jReporter.forRegistry( registry ).outputTo( logger )
- .convertRatesTo( TimeUnit.SECONDS )
- .convertDurationsTo( TimeUnit.MILLISECONDS ).build();
-
- reporter.start( 10, TimeUnit.SECONDS );
+ logger.debug( "Starting metrics reporting" );
}
@After
public void printReport() {
- logger.debug("Printing metrics report");
- reporter.report();
- reporter.stop();
+ logger.debug( "Printing metrics report" );
}
- @Test
+ @Test( timeout = 120000 )
public void rebuildOneCollectionIndex() throws Exception {
- logger.info("Started rebuildIndex()");
+ logger.info( "Started rebuildIndex()" );
- String rand = RandomStringUtils.randomAlphanumeric(5);
- final UUID appId = setup.createApplication("org_" + rand, "app_" + rand);
+ String rand = RandomStringUtils.randomAlphanumeric( 5 );
+ final UUID appId = setup.createApplication( "org_" + rand, "app_" + rand );
final EntityManager em = setup.getEmf().getEntityManager( appId );
@@ -102,109 +91,80 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT {
// ----------------- create a bunch of entities
Map<String, Object> entityMap = new HashMap<String, Object>() {{
- put("key1", 1000 );
- put("key2", 2000 );
- put("key3", "Some value");
+ put( "key1", 1000 );
+ put( "key2", 2000 );
+ put( "key3", "Some value" );
}};
List<EntityRef> entityRefs = new ArrayList<EntityRef>();
- int herderCount = 0;
+ int herderCount = 0;
int shepardCount = 0;
- for (int i = 0; i < ENTITIES_TO_INDEX; i++) {
+ for ( int i = 0; i < ENTITIES_TO_INDEX; i++ ) {
final Entity entity;
try {
- entityMap.put("key", i );
+ entityMap.put( "key", i );
if ( i % 2 == 0 ) {
- entity = em.create("catherder", entityMap);
+ entity = em.create( "catherder", entityMap );
herderCount++;
- } else {
- entity = em.create("catshepard", entityMap);
+ }
+ else {
+ entity = em.create( "catshepard", entityMap );
shepardCount++;
}
-
- app.refreshIndex();
-
-// em.createConnection(entity, "herds", cat1);
-// em.createConnection(entity, "herds", cat2);
-// em.createConnection(entity, "herds", cat3);
-
- } catch (Exception ex) {
- throw new RuntimeException("Error creating entity", ex);
+ }
+ catch ( Exception ex ) {
+ throw new RuntimeException( "Error creating entity", ex );
}
- entityRefs.add(new SimpleEntityRef( entity.getType(), entity.getUuid() ) );
+ entityRefs.add( new SimpleEntityRef( entity.getType(), entity.getUuid() ) );
if ( i % 10 == 0 ) {
- logger.info("Created {} entities", i );
+ logger.info( "Created {} entities", i );
}
-
}
- logger.info("Created {} entities", ENTITIES_TO_INDEX);
+ logger.info( "Created {} entities", ENTITIES_TO_INDEX );
app.refreshIndex();
// ----------------- test that we can read them, should work fine
- logger.debug("Read the data");
- readData( em, "catherders", herderCount, 0);
- readData( em, "catshepards", shepardCount, 0);
+ logger.debug( "Read the data" );
+ readData( em, "catherders", herderCount, 0 );
+ readData( em, "catshepards", shepardCount, 0 );
// ----------------- delete the system and application indexes
- logger.debug("Deleting apps");
+ logger.debug( "Deleting apps" );
deleteIndex( em.getApplicationId() );
// ----------------- test that we can read them, should fail
- logger.debug("Reading data, should fail this time ");
- try {
- readData( em, "testTypes", ENTITIES_TO_INDEX, 0 );
- fail("should have failed to read data");
+ logger.debug( "Reading data, should fail this time " );
- } catch (Exception expected) {}
+ //should be no data
+ readData( em, "testTypes", 0, 0 );
-// ----------------- rebuild index for catherders only
- logger.debug("Preparing to rebuild all indexes");;
+ // ----------------- rebuild index for catherders only
- final String meterName = this.getClass().getSimpleName() + ".rebuildIndex";
- final Meter meter = registry.meter( meterName );
+ logger.debug( "Preparing to rebuild all indexes" );
- EntityManagerFactory.ProgressObserver po = new EntityManagerFactory.ProgressObserver() {
- int counter = 0;
- @Override
- public void onProgress( final EntityRef entity ) {
-
- meter.mark();
- logger.debug("Indexing {}:{}", entity.getType(), entity.getUuid());
- if ( counter % 100 == 0 ) {
- logger.info("Reindexed {} entities", counter );
- }
- counter++;
- }
+ final IndexServiceRequestBuilder builder =
+ reIndexService.getBuilder().withApplicationId( em.getApplicationId() ).withCollection( "catherders" );
+ ReIndexService.ReIndexStatus status = reIndexService.rebuildIndex( builder );
+ assertNotNull( status.getJobId(), "JobId is present" );
- };
-
- try {
+ logger.info( "Rebuilt index" );
- final IndexServiceRequestBuilder builder = reIndexService.getBuilder().withApplicationId( em.getApplicationId() ).withCollection( "catherders" );
- reIndexService.rebuildIndex(builder );
+ waitForRebuild( status, reIndexService );
- reporter.report();
- registry.remove( meterName );
- logger.info("Rebuilt index");
-
- } catch (Exception ex) {
- logger.error("Error rebuilding index", ex);
- fail();
- }
// ----------------- test that we can read the catherder collection and not the catshepard
@@ -213,78 +173,79 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT {
}
- @Test
+ @Test( timeout = 120000 )
public void rebuildIndex() throws Exception {
- logger.info("Started rebuildIndex()");
+ logger.info( "Started rebuildIndex()" );
+
+ String rand = RandomStringUtils.randomAlphanumeric( 5 );
+ final UUID appId = setup.createApplication( "org_" + rand, "app_" + rand );
- String rand = RandomStringUtils.randomAlphanumeric(5);
- final UUID appId = setup.createApplication("org_" + rand, "app_" + rand);
+ final EntityManager em = setup.getEmf().getEntityManager( appId );
- final EntityManager em = setup.getEmf().getEntityManager(appId);
+ final ReIndexService reIndexService = setup.getInjector().getInstance( ReIndexService.class );
// ----------------- create a bunch of entities
Map<String, Object> entityMap = new HashMap<String, Object>() {{
- put("key1", 1000 );
- put("key2", 2000 );
- put("key3", "Some value");
+ put( "key1", 1000 );
+ put( "key2", 2000 );
+ put( "key3", "Some value" );
}};
Map<String, Object> cat1map = new HashMap<String, Object>() {{
- put("name", "enzo");
- put("color", "orange");
+ put( "name", "enzo" );
+ put( "color", "orange" );
}};
Map<String, Object> cat2map = new HashMap<String, Object>() {{
- put("name", "marquee");
- put("color", "grey");
+ put( "name", "marquee" );
+ put( "color", "grey" );
}};
Map<String, Object> cat3map = new HashMap<String, Object>() {{
- put("name", "bertha");
- put("color", "tabby");
+ put( "name", "bertha" );
+ put( "color", "tabby" );
}};
- Entity cat1 = em.create("cat", cat1map );
- Entity cat2 = em.create("cat", cat2map );
- Entity cat3 = em.create("cat", cat3map );
+ Entity cat1 = em.create( "cat", cat1map );
+ Entity cat2 = em.create( "cat", cat2map );
+ Entity cat3 = em.create( "cat", cat3map );
- List<EntityRef> entityRefs = new ArrayList<EntityRef>();
- int entityCount = 0;
- for (int i = 0; i < ENTITIES_TO_INDEX; i++) {
+ List<EntityRef> entityRefs = new ArrayList<>();
+
+ for ( int i = 0; i < ENTITIES_TO_INDEX; i++ ) {
final Entity entity;
try {
- entityMap.put("key", entityCount );
- entity = em.create("testType", entityMap );
+ entityMap.put( "key", i );
+ entity = em.create( "testType", entityMap );
- em.createConnection(entity, "herds", cat1);
- em.createConnection(entity, "herds", cat2);
- em.createConnection(entity, "herds", cat3);
-
- } catch (Exception ex) {
- throw new RuntimeException("Error creating entity", ex);
+ em.createConnection( entity, "herds", cat1 );
+ em.createConnection( entity, "herds", cat2 );
+ em.createConnection( entity, "herds", cat3 );
}
-
- entityRefs.add(new SimpleEntityRef( entity.getType(), entity.getUuid() ) );
- if ( entityCount % 10 == 0 ) {
- logger.info("Created {} entities", entityCount );
+ catch ( Exception ex ) {
+ throw new RuntimeException( "Error creating entity", ex );
}
+ entityRefs.add( new SimpleEntityRef( entity.getType(), entity.getUuid() ) );
+ if ( i % 10 == 0 ) {
+ logger.info( "Created {} entities", i );
+ }
}
- logger.info("Created {} entities", entityCount);
+ logger.info( "Created {} entities", ENTITIES_TO_INDEX );
app.refreshIndex();
- Thread.sleep(10000);
// ----------------- test that we can read them, should work fine
- logger.debug("Read the data");
- readData( em, "testType", entityCount, 3 );
+ logger.debug( "Read the data" );
+ final String collectionName = "testtypes";
+ readData( em, collectionName, ENTITIES_TO_INDEX, 3 );
// ----------------- delete the system and application indexes
- logger.debug("Deleting app index");
+ logger.debug( "Deleting app index" );
deleteIndex( em.getApplicationId() );
@@ -295,61 +256,73 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT {
// ----------------- test that we can read them, should fail
- logger.debug("Reading data, should fail this time ");
- try {
- readData( em, "testTypes", entityCount, 3 );
- fail("should have failed to read data");
+ logger.debug( "Reading data, should fail this time " );
+
+ readData( em, collectionName, 0, 0 );
+
- } catch (Exception expected) {}
// ----------------- rebuild index
- logger.debug("Preparing to rebuild all indexes");;
+ logger.debug( "Preparing to rebuild all indexes" );
+ ;
- final String meterName = this.getClass().getSimpleName() + ".rebuildIndex";
- final Meter meter = registry.meter( meterName );
- EntityManagerFactory.ProgressObserver po = new EntityManagerFactory.ProgressObserver() {
- int counter = 0;
+ try {
- @Override
- public void onProgress( final EntityRef entity ) {
+ final IndexServiceRequestBuilder builder =
+ reIndexService.getBuilder().withApplicationId( em.getApplicationId() );
- meter.mark();
- logger.debug("Indexing {}:{}", entity.getType(), entity.getUuid());
- if ( counter % 100 == 0 ) {
- logger.info("Reindexed {} entities", counter );
- }
- counter++;
- }
+ ReIndexService.ReIndexStatus status = reIndexService.rebuildIndex( builder );
+ assertNotNull( status.getJobId(), "JobId is present" );
- };
+ logger.info( "Rebuilt index" );
- try {
- fail( "Implement index rebuild" );
-// setup.getEmf().rebuildInternalIndexes( po );
-//
-// setup.getEmf().rebuildApplicationIndexes( em.getApplicationId(), po );
+ waitForRebuild( status, reIndexService );
- reporter.report();
- registry.remove( meterName );
- logger.info("Rebuilt index");
- app.refreshIndex();
+ logger.info( "Rebuilt index" );
- } catch (Exception ex) {
- logger.error("Error rebuilding index", ex);
+ app.refreshIndex();
+ }
+ catch ( Exception ex ) {
+ logger.error( "Error rebuilding index", ex );
fail();
}
// ----------------- test that we can read them
- Thread.sleep(2000);
- readData( em, "testTypes", entityCount, 3 );
+ Thread.sleep( 2000 );
+ readData( em, collectionName, ENTITIES_TO_INDEX, 3 );
+ }
+
+
+ /**
+ * Wait for the rebuild to occur
+ */
+ private void waitForRebuild( final ReIndexService.ReIndexStatus status, final ReIndexService reIndexService )
+ throws InterruptedException {
+ while ( true ) {
+
+ try {
+ final ReIndexService.ReIndexStatus updatedStatus = reIndexService.getStatus( status.getJobId() );
+
+ if ( updatedStatus.getStatus() == ReIndexService.Status.COMPLETE ) {
+ break;
+ }
+ }
+ catch ( IllegalArgumentException iae ) {
+ //swallow
+ }
+
+
+ Thread.sleep( 1000 );
+ }
}
+
/**
* Delete app index
*/
@@ -360,54 +333,49 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT {
Id appId = new SimpleId( appUuid, Schema.TYPE_APPLICATION );
ApplicationScope scope = new ApplicationScopeImpl( appId );
- ApplicationEntityIndex ei = eif.createApplicationEntityIndex(scope);
+ ApplicationEntityIndex ei = eif.createApplicationEntityIndex( scope );
- ei.deleteApplication().toBlocking().lastOrDefault(null);
+ ei.deleteApplication().toBlocking().lastOrDefault( null );
app.refreshIndex();
-
}
- private int readData( EntityManager em,
- String collectionName, int expectedEntities, int expectedConnections ) throws Exception {
+ private int readData( EntityManager em, String collectionName, int expectedEntities, int expectedConnections )
+ throws Exception {
app.refreshIndex();
- Query q = Query.fromQL("select * where key1=1000");
- q.setLimit(40);
- Results results = em.searchCollectionConsistent( em.getApplicationRef(), collectionName, q,expectedEntities );
+ Query q = Query.fromQL( "select * where key1=1000" ).withLimit( 1000 );
+ Results results = em.searchCollectionConsistent( em.getApplicationRef(), collectionName, q, expectedEntities );
int count = 0;
while ( true ) {
for ( Entity e : results.getEntities() ) {
- assertEquals( 2000, e.getProperty("key2"));
+ assertEquals( 2000, e.getProperty( "key2" ) );
- Results catResults = em.searchTargetEntities(e,
- Query.fromQL("select *").setConnectionType("herds"));
+ Results catResults =
+ em.searchTargetEntities( e, Query.fromQL( "select *" ).setConnectionType( "herds" ) );
assertEquals( expectedConnections, catResults.size() );
if ( count % 100 == 0 ) {
- logger.info( "read {} entities", count);
+ logger.info( "read {} entities", count );
}
count++;
}
if ( results.hasCursor() ) {
- logger.info( "Counted {} : query again with cursor", count);
+ logger.info( "Counted {} : query again with cursor", count );
q.setCursor( results.getCursor() );
results = em.searchCollection( em.getApplicationRef(), collectionName, q );
-
- } else {
+ }
+ else {
break;
}
}
- if ( expectedEntities != -1 && expectedEntities != count ) {
- throw new RuntimeException("Did not get expected "
- + expectedEntities + " entities, instead got " + count );
- }
+ assertEquals("Did not get expected entities", expectedEntities, count);
return count;
}
}
[20/22] incubator-usergrid git commit: Merge remote-tracking branch
'origin/USERGRID-643' into two-dot-o-dev
Posted by gr...@apache.org.
Merge remote-tracking branch 'origin/USERGRID-643' into two-dot-o-dev
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/b28aeee0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/b28aeee0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/b28aeee0
Branch: refs/heads/USERGRID-608
Commit: b28aeee0522a3ebef8e2d349676ea381baf3ee37
Parents: b637535 53aa87c
Author: GERey <gr...@apigee.com>
Authored: Wed May 20 16:00:15 2015 -0700
Committer: GERey <gr...@apigee.com>
Committed: Wed May 20 16:00:15 2015 -0700
----------------------------------------------------------------------
.../usergrid/corepersistence/CoreModule.java | 4 +
.../corepersistence/CpEntityManagerFactory.java | 14 -
.../corepersistence/CpRelationManager.java | 1 -
.../asyncevents/EventBuilder.java | 6 +-
.../asyncevents/EventBuilderImpl.java | 18 +-
.../asyncevents/InMemoryAsyncEventService.java | 7 +-
.../asyncevents/SQSAsyncEventService.java | 9 +-
.../index/EdgeScopeSerializer.java | 41 +++
.../index/EntityIndexOperation.java | 46 +++
.../index/IndexProcessorFig.java | 6 +-
.../corepersistence/index/ReIndexAction.java | 6 +-
.../index/ReIndexRequestBuilder.java | 86 +++++
.../index/ReIndexRequestBuilderImpl.java | 122 +++++++
.../corepersistence/index/ReIndexService.java | 89 +++--
.../index/ReIndexServiceImpl.java | 251 ++++++++++++--
.../cursor/AbstractCursorSerializer.java | 2 +-
.../pipeline/cursor/CursorSerializerUtil.java | 54 ++-
.../pipeline/cursor/RequestCursor.java | 9 +-
.../pipeline/cursor/ResponseCursor.java | 49 +--
.../pipeline/read/AbstractPathFilter.java | 30 --
.../pipeline/read/CursorSeek.java | 53 +++
.../rx/impl/AllEntityIdsObservable.java | 5 +-
.../rx/impl/AllEntityIdsObservableImpl.java | 6 +-
.../util/SerializableMapper.java | 91 -----
.../persistence/EntityManagerFactory.java | 2 -
.../rx/EdgesToTargetObservableIT.java | 4 +-
.../PerformanceEntityRebuildIndexTest.java | 346 ++++++++-----------
.../data/MigrationInfoSerializationImpl.java | 4 +-
.../migration/schema/MigrationManagerImpl.java | 14 +-
.../core/migration/util/AstayanxUtils.java | 49 +++
.../graph/serialization/EdgesObservable.java | 24 +-
.../serialization/impl/EdgesObservableImpl.java | 10 +-
.../impl/TargetIdObservableImpl.java | 2 +-
.../impl/migration/EdgeDataMigrationImpl.java | 2 +-
.../persistence/index/impl/IndexingUtils.java | 2 +
stack/pom.xml | 13 +
.../org/apache/usergrid/rest/IndexResource.java | 342 +++++++++---------
.../main/resources/usergrid-rest-context.xml | 3 -
.../resources/usergrid-rest-deploy-context.xml | 1 -
39 files changed, 1138 insertions(+), 685 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b28aeee0/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b28aeee0/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/InMemoryAsyncEventService.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b28aeee0/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgesObservableImpl.java
----------------------------------------------------------------------
[17/22] incubator-usergrid git commit: Updates REST api calls. POST
is used to create a re-index job, PUT is used to resume.
Posted by gr...@apache.org.
Updates REST api calls. POST is used to create a re-index job, PUT is used to resume.
Also fixes transitive dependency issue with clouds
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/5a7f9c09
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/5a7f9c09
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/5a7f9c09
Branch: refs/heads/USERGRID-608
Commit: 5a7f9c096b5d79950b5df658070ae0bd9bdc77da
Parents: 48be894
Author: Todd Nine <tn...@apigee.com>
Authored: Fri May 15 19:24:17 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Fri May 15 19:24:17 2015 -0600
----------------------------------------------------------------------
.../index/IndexServiceRequestBuilder.java | 88 -----
.../index/IndexServiceRequestBuilderImpl.java | 122 -------
.../index/ReIndexRequestBuilder.java | 86 +++++
.../index/ReIndexRequestBuilderImpl.java | 122 +++++++
.../corepersistence/index/ReIndexService.java | 6 +-
.../index/ReIndexServiceImpl.java | 14 +-
.../PerformanceEntityRebuildIndexTest.java | 6 +-
.../persistence/index/impl/IndexingUtils.java | 2 +
stack/pom.xml | 13 +
.../org/apache/usergrid/rest/IndexResource.java | 342 +++++++++----------
.../main/resources/usergrid-rest-context.xml | 3 -
.../resources/usergrid-rest-deploy-context.xml | 1 -
12 files changed, 401 insertions(+), 404 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5a7f9c09/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceRequestBuilder.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceRequestBuilder.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceRequestBuilder.java
deleted file mode 100644
index 07160d8..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceRequestBuilder.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.index;
-
-
-import java.util.UUID;
-
-import org.elasticsearch.action.index.IndexRequestBuilder;
-
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-
-import com.google.common.base.Optional;
-
-
-/**
- * A builder interface to build our re-index request
- */
-public interface IndexServiceRequestBuilder {
-
- /**
- * Set the application id
- */
- IndexServiceRequestBuilder withApplicationId( final UUID applicationId );
-
- /**
- * Set the collection name. If not set, every collection will be reindexed
- * @param collectionName
- * @return
- */
- IndexServiceRequestBuilder withCollection( final String collectionName );
-
- /**
- * Set our cursor to resume processing
- * @param cursor
- * @return
- */
- IndexServiceRequestBuilder withCursor(final String cursor);
-
-
- /**
- * Set the timestamp to re-index entities updated >= this timestamp
- * @param timestamp
- * @return
- */
- IndexServiceRequestBuilder withStartTimestamp(final Long timestamp);
-
-
- /**
- * Get the application scope
- * @return
- */
- Optional<ApplicationScope> getApplicationScope();
-
- /**
- * Get the collection name
- * @return
- */
- Optional<String> getCollectionName();
-
- /**
- * Get the cursor
- * @return
- */
- Optional<String> getCursor();
-
- /**
- * Get the updated since timestamp
- * @return
- */
- Optional<Long> getUpdateTimestamp();
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5a7f9c09/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceRequestBuilderImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceRequestBuilderImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceRequestBuilderImpl.java
deleted file mode 100644
index 4017b6e..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceRequestBuilderImpl.java
+++ /dev/null
@@ -1,122 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.index;
-
-
-import java.util.UUID;
-
-import org.apache.usergrid.corepersistence.util.CpNamingUtils;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-
-import com.google.common.base.Optional;
-
-
-/**
- * Index service request builder
- */
-public class IndexServiceRequestBuilderImpl implements IndexServiceRequestBuilder {
-
- private Optional<UUID> withApplicationId = Optional.absent();
- private Optional<String> withCollectionName = Optional.absent();
- private Optional<String> cursor = Optional.absent();
- private Optional<Long> updateTimestamp = Optional.absent();
-
-
- /***
- *
- * @param applicationId The application id
- * @return
- */
- @Override
- public IndexServiceRequestBuilder withApplicationId( final UUID applicationId ) {
- this.withApplicationId = Optional.fromNullable( applicationId );
- return this;
- }
-
-
- /**
- * the colleciton name
- * @param collectionName
- * @return
- */
- @Override
- public IndexServiceRequestBuilder withCollection( final String collectionName ) {
- if(collectionName == null){
- this.withCollectionName = Optional.absent();
- }
- else {
- this.withCollectionName = Optional.fromNullable( CpNamingUtils.getEdgeTypeFromCollectionName( collectionName ) );
- }
- return this;
- }
-
-
- /**
- * The cursor
- * @param cursor
- * @return
- */
- @Override
- public IndexServiceRequestBuilder withCursor( final String cursor ) {
- this.cursor = Optional.fromNullable( cursor );
- return this;
- }
-
-
- /**
- * Set start timestamp in epoch time. Only entities updated since this time will be processed for indexing
- * @param timestamp
- * @return
- */
- @Override
- public IndexServiceRequestBuilder withStartTimestamp( final Long timestamp ) {
- this.updateTimestamp = Optional.fromNullable( timestamp );
- return this;
- }
-
-
- @Override
- public Optional<ApplicationScope> getApplicationScope() {
-
- if ( this.withApplicationId.isPresent() ) {
- return Optional.of( CpNamingUtils.getApplicationScope( withApplicationId.get() ) );
- }
-
- return Optional.absent();
- }
-
-
- @Override
- public Optional<String> getCollectionName() {
- return withCollectionName;
- }
-
-
- @Override
- public Optional<String> getCursor() {
- return cursor;
- }
-
-
- @Override
- public Optional<Long> getUpdateTimestamp() {
- return updateTimestamp;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5a7f9c09/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexRequestBuilder.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexRequestBuilder.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexRequestBuilder.java
new file mode 100644
index 0000000..0863a63
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexRequestBuilder.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.index;
+
+
+import java.util.UUID;
+
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+
+import com.google.common.base.Optional;
+
+
+/**
+ * A builder interface to build our re-index request
+ */
+public interface ReIndexRequestBuilder {
+
+ /**
+ * Set the application id
+ */
+ ReIndexRequestBuilder withApplicationId( final UUID applicationId );
+
+ /**
+ * Set the collection name. If not set, every collection will be reindexed
+ * @param collectionName
+ * @return
+ */
+ ReIndexRequestBuilder withCollection( final String collectionName );
+
+ /**
+ * Set our cursor to resume processing
+ * @param cursor
+ * @return
+ */
+ ReIndexRequestBuilder withCursor(final String cursor);
+
+
+ /**
+ * Set the timestamp to re-index entities updated >= this timestamp
+ * @param timestamp
+ * @return
+ */
+ ReIndexRequestBuilder withStartTimestamp(final Long timestamp);
+
+
+ /**
+ * Get the application scope
+ * @return
+ */
+ Optional<ApplicationScope> getApplicationScope();
+
+ /**
+ * Get the collection name
+ * @return
+ */
+ Optional<String> getCollectionName();
+
+ /**
+ * Get the cursor
+ * @return
+ */
+ Optional<String> getCursor();
+
+ /**
+ * Get the updated since timestamp
+ * @return
+ */
+ Optional<Long> getUpdateTimestamp();
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5a7f9c09/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexRequestBuilderImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexRequestBuilderImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexRequestBuilderImpl.java
new file mode 100644
index 0000000..25e71e6
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexRequestBuilderImpl.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.index;
+
+
+import java.util.UUID;
+
+import org.apache.usergrid.corepersistence.util.CpNamingUtils;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+
+import com.google.common.base.Optional;
+
+
+/**
+ * Index service request builder
+ */
+public class ReIndexRequestBuilderImpl implements ReIndexRequestBuilder {
+
+ private Optional<UUID> withApplicationId = Optional.absent();
+ private Optional<String> withCollectionName = Optional.absent();
+ private Optional<String> cursor = Optional.absent();
+ private Optional<Long> updateTimestamp = Optional.absent();
+
+
+ /***
+ *
+ * @param applicationId The application id
+ * @return
+ */
+ @Override
+ public ReIndexRequestBuilder withApplicationId( final UUID applicationId ) {
+ this.withApplicationId = Optional.fromNullable( applicationId );
+ return this;
+ }
+
+
+ /**
+ * the colleciton name
+ * @param collectionName
+ * @return
+ */
+ @Override
+ public ReIndexRequestBuilder withCollection( final String collectionName ) {
+ if(collectionName == null){
+ this.withCollectionName = Optional.absent();
+ }
+ else {
+ this.withCollectionName = Optional.fromNullable( CpNamingUtils.getEdgeTypeFromCollectionName( collectionName ) );
+ }
+ return this;
+ }
+
+
+ /**
+ * The cursor
+ * @param cursor
+ * @return
+ */
+ @Override
+ public ReIndexRequestBuilder withCursor( final String cursor ) {
+ this.cursor = Optional.fromNullable( cursor );
+ return this;
+ }
+
+
+ /**
+ * Set start timestamp in epoch time. Only entities updated since this time will be processed for indexing
+ * @param timestamp
+ * @return
+ */
+ @Override
+ public ReIndexRequestBuilder withStartTimestamp( final Long timestamp ) {
+ this.updateTimestamp = Optional.fromNullable( timestamp );
+ return this;
+ }
+
+
+ @Override
+ public Optional<ApplicationScope> getApplicationScope() {
+
+ if ( this.withApplicationId.isPresent() ) {
+ return Optional.of( CpNamingUtils.getApplicationScope( withApplicationId.get() ) );
+ }
+
+ return Optional.absent();
+ }
+
+
+ @Override
+ public Optional<String> getCollectionName() {
+ return withCollectionName;
+ }
+
+
+ @Override
+ public Optional<String> getCursor() {
+ return cursor;
+ }
+
+
+ @Override
+ public Optional<Long> getUpdateTimestamp() {
+ return updateTimestamp;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5a7f9c09/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java
index af3615e..bae8d1f 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java
@@ -29,15 +29,15 @@ public interface ReIndexService {
/**
* Perform an index rebuild
*
- * @param indexServiceRequestBuilder The builder to build the request
+ * @param reIndexRequestBuilder The builder to build the request
*/
- ReIndexStatus rebuildIndex( final IndexServiceRequestBuilder indexServiceRequestBuilder );
+ ReIndexStatus rebuildIndex( final ReIndexRequestBuilder reIndexRequestBuilder );
/**
* Generate a build for the index
*/
- IndexServiceRequestBuilder getBuilder();
+ ReIndexRequestBuilder getBuilder();
/**
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5a7f9c09/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
index f44113b..ef03866 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
@@ -91,16 +91,16 @@ public class ReIndexServiceImpl implements ReIndexService {
@Override
- public ReIndexStatus rebuildIndex( final IndexServiceRequestBuilder indexServiceRequestBuilder ) {
+ public ReIndexStatus rebuildIndex( final ReIndexRequestBuilder reIndexRequestBuilder ) {
//load our last emitted Scope if a cursor is present
- final Optional<EdgeScope> cursor = parseCursor( indexServiceRequestBuilder.getCursor() );
+ final Optional<EdgeScope> cursor = parseCursor( reIndexRequestBuilder.getCursor() );
final CursorSeek<Edge> cursorSeek = getResumeEdge( cursor );
- final Optional<ApplicationScope> appId = indexServiceRequestBuilder.getApplicationScope();
+ final Optional<ApplicationScope> appId = reIndexRequestBuilder.getApplicationScope();
Preconditions.checkArgument( !(cursor.isPresent() && appId.isPresent()),
@@ -111,11 +111,11 @@ public class ReIndexServiceImpl implements ReIndexService {
final String jobId = StringUtils.sanitizeUUID( UUIDGenerator.newTimeUUID() );
- final long modifiedSince = indexServiceRequestBuilder.getUpdateTimestamp().or( Long.MIN_VALUE );
+ final long modifiedSince = reIndexRequestBuilder.getUpdateTimestamp().or( Long.MIN_VALUE );
//create an observable that loads each entity and indexes it, start it running with publish
final Observable<EdgeScope> runningReIndex = allEntityIdsObservable.getEdgesToEntities( applicationScopes,
- indexServiceRequestBuilder.getCollectionName(), cursorSeek.getSeekValue() )
+ reIndexRequestBuilder.getCollectionName(), cursorSeek.getSeekValue() )
//for each edge, create our scope and index on it
.doOnNext( edge -> {
@@ -143,8 +143,8 @@ public class ReIndexServiceImpl implements ReIndexService {
@Override
- public IndexServiceRequestBuilder getBuilder() {
- return new IndexServiceRequestBuilderImpl();
+ public ReIndexRequestBuilder getBuilder() {
+ return new ReIndexRequestBuilderImpl();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5a7f9c09/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java b/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
index 8d54043..06eb060 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
@@ -33,7 +33,7 @@ import org.apache.commons.lang.RandomStringUtils;
import org.apache.usergrid.AbstractCoreIT;
import org.apache.usergrid.cassandra.SpringResource;
-import org.apache.usergrid.corepersistence.index.IndexServiceRequestBuilder;
+import org.apache.usergrid.corepersistence.index.ReIndexRequestBuilder;
import org.apache.usergrid.corepersistence.index.ReIndexService;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
@@ -153,7 +153,7 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT {
logger.debug( "Preparing to rebuild all indexes" );
- final IndexServiceRequestBuilder builder =
+ final ReIndexRequestBuilder builder =
reIndexService.getBuilder().withApplicationId( em.getApplicationId() ).withCollection( "catherders" );
ReIndexService.ReIndexStatus status = reIndexService.rebuildIndex( builder );
@@ -270,7 +270,7 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT {
try {
- final IndexServiceRequestBuilder builder =
+ final ReIndexRequestBuilder builder =
reIndexService.getBuilder().withApplicationId( em.getApplicationId() );
ReIndexService.ReIndexStatus status = reIndexService.rebuildIndex( builder );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5a7f9c09/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
index 8b248aa..bc15149 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
@@ -99,6 +99,8 @@ public class IndexingUtils {
/**
* Create our sub scope. This is the ownerUUID + type
+ *
+ * TODO make this format more readable and parsable
*/
public static String createContextName( final ApplicationScope applicationScope, final SearchEdge scope ) {
StringBuilder sb = new StringBuilder();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5a7f9c09/stack/pom.xml
----------------------------------------------------------------------
diff --git a/stack/pom.xml b/stack/pom.xml
index 762607d..4559db2 100644
--- a/stack/pom.xml
+++ b/stack/pom.xml
@@ -1241,6 +1241,13 @@
<groupId>org.apache.jclouds</groupId>
<artifactId>jclouds-blobstore</artifactId>
<version>${jclouds.version}</version>
+ <exclusions>
+ <!-- blows up our version of guice-->
+ <exclusion>
+ <groupId>com.google.inject.extensions</groupId>
+ <artifactId>guice-assistedinject</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
@@ -1262,6 +1269,12 @@
<groupId>aopalliance</groupId>
<artifactId>aopalliance</artifactId>
</exclusion>
+
+ <!-- blows up our version of guice-->
+ <exclusion>
+ <groupId>com.google.inject.extensions</groupId>
+ <artifactId>guice-assistedinject</artifactId>
+ </exclusion>
</exclusions>
</dependency>
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5a7f9c09/stack/rest/src/main/java/org/apache/usergrid/rest/IndexResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/IndexResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/IndexResource.java
index 668c05c..ffb9700 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/IndexResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/IndexResource.java
@@ -20,25 +20,34 @@
package org.apache.usergrid.rest;
-import com.google.common.base.Optional;
-import com.google.common.base.Preconditions;
-import com.sun.jersey.api.json.JSONWithPadding;
-import org.apache.usergrid.persistence.EntityManagerFactory;
-import org.apache.usergrid.persistence.EntityRef;
-import org.apache.usergrid.persistence.index.utils.UUIDUtils;
-import org.apache.usergrid.rest.security.annotations.RequireSystemAccess;
+
+import java.util.Map;
+import java.util.UUID;
+
+import javax.ws.rs.DefaultValue;
+import javax.ws.rs.POST;
+import javax.ws.rs.PUT;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.UriInfo;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component;
-import javax.ws.rs.*;
-import javax.ws.rs.core.Context;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.UriInfo;
-import java.util.Map;
-import java.util.Set;
-import java.util.UUID;
+import org.apache.usergrid.corepersistence.index.ReIndexRequestBuilder;
+import org.apache.usergrid.corepersistence.index.ReIndexService;
+import org.apache.usergrid.persistence.index.utils.UUIDUtils;
+import org.apache.usergrid.rest.security.annotations.RequireSystemAccess;
+
+import com.google.common.base.Preconditions;
+import com.sun.jersey.api.json.JSONWithPadding;
+
/**
* Classy class class.
@@ -46,260 +55,239 @@ import java.util.UUID;
@Component
@Scope( "singleton" )
@Produces( {
- MediaType.APPLICATION_JSON, "application/javascript", "application/x-javascript", "text/ecmascript",
- "application/ecmascript", "text/jscript"
+ MediaType.APPLICATION_JSON, "application/javascript", "application/x-javascript", "text/ecmascript",
+ "application/ecmascript", "text/jscript"
} )
public class IndexResource extends AbstractContextResource {
- private static final Logger logger = LoggerFactory.getLogger(IndexResource.class);
+ private static final Logger logger = LoggerFactory.getLogger( IndexResource.class );
+ private static final String UPDATED_FIELD = "updated";
- public IndexResource(){
+
+ public IndexResource() {
super();
}
+
@RequireSystemAccess
- @PUT
+ @POST
@Path( "rebuild" )
- public JSONWithPadding rebuildIndexes( @Context UriInfo ui,
- @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
- throws Exception {
+ public JSONWithPadding rebuildIndexesPost( @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+ throws Exception {
- ApiResponse response = createApiResponse();
- response.setAction( "rebuild indexes" );
+ logger.info( "Rebuilding all applications" );
- final EntityManagerFactory.ProgressObserver po = new EntityManagerFactory.ProgressObserver() {
+ final ReIndexRequestBuilder request = createRequest();
+ return executeAndCreateResponse( request, callback );
+ }
- @Override
- public void onProgress( final EntityRef entity ) {
- logger.info( "Indexing entity {}:{} ", entity.getType(), entity.getUuid() );
- }
- };
+ @RequireSystemAccess
+ @PUT
+ @Path( "rebuild" )
+ public JSONWithPadding rebuildIndexesPut( final Map<String, Object> payload,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+ throws Exception {
- final Thread rebuild = new Thread() {
+ logger.info( "Resuming rebuilding all applications" );
+ final ReIndexRequestBuilder request = createRequest();
- @Override
- public void run() {
- logger.info( "Rebuilding all indexes" );
+ return executeResumeAndCreateResponse( payload, request, callback );
+ }
- try {
- emf.rebuildAllIndexes( po );
- }
- catch ( Exception e ) {
- logger.error( "Unable to rebuild indexes", e );
- }
- logger.info( "Completed all indexes" );
- }
- };
+ @RequireSystemAccess
+ @POST
+ @Path( "rebuild/" + RootResource.APPLICATION_ID_PATH )
+ public JSONWithPadding rebuildIndexesPut( @PathParam( "applicationId" ) String applicationIdStr,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback,
+ @QueryParam( "delay" ) @DefaultValue( "10" ) final long delay )
- rebuild.setName( "Index rebuild all usergrid" );
- rebuild.setDaemon( true );
- rebuild.start();
+ throws Exception {
- response.setSuccess();
+ logger.info( "Rebuilding application {}", applicationIdStr );
- return new JSONWithPadding( response, callback );
+
+ final UUID appId = UUIDUtils.tryExtractUUID( applicationIdStr );
+
+ final ReIndexRequestBuilder request = createRequest().withApplicationId( appId );
+
+ return executeAndCreateResponse( request, callback );
}
@RequireSystemAccess
@PUT
@Path( "rebuild/" + RootResource.APPLICATION_ID_PATH )
- public JSONWithPadding rebuildIndexes( @Context UriInfo ui, @PathParam( "applicationId" ) String applicationIdStr,
- @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback,
- @QueryParam( "delay" ) @DefaultValue( "10" ) final long delay )
-
- throws Exception {
-
- final UUID appId = UUIDUtils.tryExtractUUID(applicationIdStr);
- ApiResponse response = createApiResponse();
- response.setAction( "rebuild indexes started" );
+ public JSONWithPadding rebuildIndexesPut( final Map<String, Object> payload,
+ @PathParam( "applicationId" ) String applicationIdStr,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback,
+ @QueryParam( "delay" ) @DefaultValue( "10" ) final long delay )
- final EntityManagerFactory.ProgressObserver po = new EntityManagerFactory.ProgressObserver() {
+ throws Exception {
- @Override
- public void onProgress( final EntityRef entity ) {
- logger.info( "Indexing entity {}:{}", entity.getType(), entity.getUuid() );
- }
+ logger.info( "Resuming rebuilding application {}", applicationIdStr );
+ final UUID appId = UUIDUtils.tryExtractUUID( applicationIdStr );
- };
-
-
- final Thread rebuild = new Thread() {
-
- @Override
- public void run() {
-
+ final ReIndexRequestBuilder request = createRequest().withApplicationId( appId );
- logger.info( "Started rebuilding application {} in collection ", appId );
+ return executeResumeAndCreateResponse( payload, request, callback );
+ }
- try {
- emf.rebuildApplicationIndexes( appId, po );
- }
- catch ( Exception e ) {
- logger.error( "Unable to re-index application", e );
- }
+ @RequireSystemAccess
+ @POST
+ @Path( "rebuild/" + RootResource.APPLICATION_ID_PATH + "/{collectionName}" )
+ public JSONWithPadding rebuildIndexesPost( @PathParam( "applicationId" ) final String applicationIdStr,
+ @PathParam( "collectionName" ) final String collectionName,
+ @QueryParam( "reverse" ) @DefaultValue( "false" ) final Boolean reverse,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+ throws Exception {
- logger.info( "Completed rebuilding application {} in collection ", appId );
- }
- };
+ logger.info( "Rebuilding collection {} in application {}", collectionName, applicationIdStr );
- rebuild.setName( String.format( "Index rebuild for app %s", appId ) );
- rebuild.setDaemon( true );
- rebuild.start();
+ final UUID appId = UUIDUtils.tryExtractUUID( applicationIdStr );
- response.setSuccess();
+ final ReIndexRequestBuilder request =
+ createRequest().withApplicationId( appId ).withCollection( collectionName );
- return new JSONWithPadding( response, callback );
+ return executeAndCreateResponse( request, callback );
}
@RequireSystemAccess
@PUT
@Path( "rebuild/" + RootResource.APPLICATION_ID_PATH + "/{collectionName}" )
- public JSONWithPadding rebuildIndexes(
- @Context UriInfo ui,
- @PathParam( "applicationId" ) final String applicationIdStr,
- @PathParam( "collectionName" ) final String collectionName,
- @QueryParam( "reverse" ) @DefaultValue( "false" ) final Boolean reverse,
- @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback) throws Exception {
+ public JSONWithPadding rebuildIndexesPut( final Map<String, Object> payload,
+ @PathParam( "applicationId" ) final String applicationIdStr,
+ @PathParam( "collectionName" ) final String collectionName,
+ @QueryParam( "reverse" ) @DefaultValue( "false" ) final Boolean reverse,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+ throws Exception {
- final UUID appId = UUIDUtils.tryExtractUUID( applicationIdStr );
- ApiResponse response = createApiResponse();
- response.setAction( "rebuild indexes" );
+ logger.info( "Resuming rebuilding collection {} in application {}", collectionName, applicationIdStr );
- final Thread rebuild = new Thread() {
+ final UUID appId = UUIDUtils.tryExtractUUID( applicationIdStr );
- public void run() {
+ final ReIndexRequestBuilder request =
+ createRequest().withApplicationId( appId ).withCollection( collectionName );
- logger.info( "Started rebuilding application {} in collection {}", appId, collectionName );
+ return executeResumeAndCreateResponse( payload, request, callback );
+ }
- try {
- rebuildCollection( appId, collectionName, reverse );
- } catch (Exception e) {
- // TODO: handle this in rebuildCollection() instead
- throw new RuntimeException("Error rebuilding collection");
- }
+ @RequireSystemAccess
+ @POST
+ @Path( "rebuild/management" )
+ public JSONWithPadding rebuildInternalIndexesPost(
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback ) throws Exception {
- logger.info( "Completed rebuilding application {} in collection {}", appId, collectionName );
- }
- };
- rebuild.setName( String.format( "Index rebuild for app %s and collection %s", appId, collectionName ) );
- rebuild.setDaemon( true );
- rebuild.start();
+ final UUID managementAppId = emf.getManagementAppId();
- response.setSuccess();
+ logger.info( "Rebuilding management application with id {} ", managementAppId );
+ final ReIndexRequestBuilder request = createRequest().withApplicationId( managementAppId );
- return new JSONWithPadding( response, callback );
+ return executeAndCreateResponse( request, callback );
}
+
@RequireSystemAccess
- @PUT
- @Path( "rebuildinternal" )
- public JSONWithPadding rebuildInternalIndexes(
- @Context UriInfo ui,
- @PathParam( "applicationId" ) String applicationIdStr,
- @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback,
- @QueryParam( "delay" ) @DefaultValue( "10" ) final long delay ) throws Exception {
+ @POST
+ @Path( "rebuild/management" )
+ public JSONWithPadding rebuildInternalIndexesPut( final Map<String, Object> payload,
+ @QueryParam( "callback" ) @DefaultValue( "callback" )
+ String callback ) throws Exception {
- final UUID appId = UUIDUtils.tryExtractUUID(applicationIdStr);
- ApiResponse response = createApiResponse();
- response.setAction( "rebuild indexes started" );
+ final UUID managementAppId = emf.getManagementAppId();
- final EntityManagerFactory.ProgressObserver po = new EntityManagerFactory.ProgressObserver() {
+ logger.info( "Resuming rebuilding management application with id {} ", managementAppId );
+ final ReIndexRequestBuilder request = createRequest().withApplicationId( managementAppId );
- @Override
- public void onProgress( final EntityRef entity ) {
- logger.info( "Indexing entity {}:{}", entity.getType(), entity.getUuid() );
- }
+ return executeResumeAndCreateResponse( payload, request, callback );
+ }
- };
- final Thread rebuild = new Thread() {
+ @RequireSystemAccess
+ @POST
+ public JSONWithPadding addIndex( @Context UriInfo ui, Map<String, Object> config,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+ throws Exception {
- @Override
- public void run() {
+ Preconditions
+ .checkNotNull( config, "Payload for config is null, please pass {replicas:int, shards:int} in body" );
- logger.info( "Started rebuilding internal indexes", appId );
+ ApiResponse response = createApiResponse();
- try {
- emf.rebuildInternalIndexes( po );
- }
- catch ( Exception e ) {
- logger.error( "Unable to re-index internals", e );
- }
+ if ( !config.containsKey( "replicas" ) || !config.containsKey( "shards" ) ||
+ !( config.get( "replicas" ) instanceof Integer ) || !( config.get( "shards" ) instanceof Integer ) ) {
+ throw new IllegalArgumentException( "body must contains 'replicas' of type int and 'shards' of type int" );
+ }
- logger.info( "Completed rebuilding internal indexes" );
- }
- };
+ if ( !config.containsKey( "indexSuffix" ) ) {
+ throw new IllegalArgumentException( "Please add an indexSuffix to your post" );
+ }
- rebuild.setName( String.format( "Index rebuild for app %s", appId ) );
- rebuild.setDaemon( true );
- rebuild.start();
- response.setSuccess();
+ emf.addIndex( config.get( "indexSuffix" ).toString(), ( int ) config.get( "shards" ),
+ ( int ) config.get( "replicas" ), ( String ) config.get( "writeConsistency" ) );
+ response.setAction( "Add index to alias" );
return new JSONWithPadding( response, callback );
}
- @RequireSystemAccess
- @POST
- @Path( RootResource.APPLICATION_ID_PATH )
- public JSONWithPadding addIndex(@Context UriInfo ui,
- Map<String, Object> config,
- @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback) throws Exception{
- Preconditions.checkNotNull(config,"Payload for config is null, please pass {replicas:int, shards:int} in body");
+ private ReIndexService getReIndexService() {
+ return injector.getInstance( ReIndexService.class );
+ }
- ApiResponse response = createApiResponse();
- if (!config.containsKey("replicas") || !config.containsKey("shards") ||
- !(config.get("replicas") instanceof Integer) || !(config.get("shards") instanceof Integer)){
- throw new IllegalArgumentException("body must contains 'replicas' of type int and 'shards' of type int");
- }
+ private ReIndexRequestBuilder createRequest() {
+ return createRequest();
+ }
- if(!config.containsKey("indexSuffix")) {
- throw new IllegalArgumentException("Please add an indexSuffix to your post");
- }
+ private JSONWithPadding executeResumeAndCreateResponse( final Map<String, Object> payload,
+ final ReIndexRequestBuilder request,
+ final String callback ) {
- emf.addIndex( config.get("indexSuffix").toString(),
- (int) config.get("shards"),(int) config.get("replicas"),(String)config.get("writeConsistency"));
- response.setAction("Add index to alias");
+ Preconditions.checkArgument( payload.containsKey( UPDATED_FIELD ),
+ "You must specified the field \"updated\" in the payload" );
- return new JSONWithPadding(response, callback);
+ //add our updated timestamp to the request
+ if ( !payload.containsKey( UPDATED_FIELD ) ) {
+ final long timestamp = ( long ) payload.get( UPDATED_FIELD );
+ request.withStartTimestamp( timestamp );
+ }
+ return executeAndCreateResponse( request, callback );
}
- private void rebuildCollection(
- final UUID applicationId,
- final String collectionName,
- final boolean reverse) throws Exception {
- EntityManagerFactory.ProgressObserver po = new EntityManagerFactory.ProgressObserver() {
+ /**
+ * Execute the request and return the response.
+ */
+ private JSONWithPadding executeAndCreateResponse( final ReIndexRequestBuilder request, final String callback ) {
- @Override
- public void onProgress( final EntityRef entity ) {
- logger.info( "Indexing entity {}:{}", entity.getType(), entity.getUuid() );
- }
- };
+ final ReIndexService.ReIndexStatus status = getReIndexService().rebuildIndex( request );
- logger.info( "Reindexing for app id: {} and collection {}", applicationId, collectionName );
+ final ApiResponse response = createApiResponse();
- emf.rebuildCollectionIndex(Optional.of(applicationId),Optional.of(collectionName));
- getEntityIndex().refreshAsync().toBlocking().first();
- }
+ response.setAction( "rebuild indexes" );
+ response.setProperty( "jobId", status.getJobId() );
+ response.setProperty( "status", status.getStatus() );
+ response.setProperty( "lastUpdatedEpoch", status.getLastUpdated() );
+ response.setProperty( "numberQueued", status.getNumberProcessed() );
+ response.setSuccess();
+ return new JSONWithPadding( response, callback );
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5a7f9c09/stack/rest/src/main/resources/usergrid-rest-context.xml
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/resources/usergrid-rest-context.xml b/stack/rest/src/main/resources/usergrid-rest-context.xml
index 5c63e72..16f4541 100644
--- a/stack/rest/src/main/resources/usergrid-rest-context.xml
+++ b/stack/rest/src/main/resources/usergrid-rest-context.xml
@@ -47,9 +47,6 @@
<property name="securityManager" ref="securityManager" />
</bean>
- <bean id="mongoServer" class="org.apache.usergrid.mongo.MongoServer"
- init-method="startServer" destroy-method="stopServer" />
-
<!-- override the security manager -->
<bean id="securityManager" class="org.apache.usergrid.rest.security.shiro.RestSecurityManager">
<property name="cacheManager" ref="cacheManager" />
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5a7f9c09/stack/rest/src/main/resources/usergrid-rest-deploy-context.xml
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/resources/usergrid-rest-deploy-context.xml b/stack/rest/src/main/resources/usergrid-rest-deploy-context.xml
index 9965dbc..e9b7ccd 100644
--- a/stack/rest/src/main/resources/usergrid-rest-deploy-context.xml
+++ b/stack/rest/src/main/resources/usergrid-rest-deploy-context.xml
@@ -33,7 +33,6 @@
<property name="locations">
<list>
<value>classpath:/usergrid-default.properties</value>
- <value>${usergrid-custom-spring-properties}</value>
</list>
</property>
</bean>
[13/22] incubator-usergrid git commit: fix serviceinvocation it
Posted by gr...@apache.org.
fix serviceinvocation it
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/c7a4b19e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/c7a4b19e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/c7a4b19e
Branch: refs/heads/USERGRID-608
Commit: c7a4b19e2ae8e3d6fc454a9d07a46eb975981482
Parents: 0c16f0e
Author: Shawn Feldman <sf...@apache.org>
Authored: Fri May 15 13:09:54 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Fri May 15 13:09:54 2015 -0600
----------------------------------------------------------------------
.../org/apache/usergrid/corepersistence/CpRelationManager.java | 5 ++++-
.../org/apache/usergrid/corepersistence/util/CpNamingUtils.java | 3 ++-
2 files changed, 6 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c7a4b19e/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index 797788e..6690346 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -913,7 +913,10 @@ public class CpRelationManager implements RelationManager {
if ( query.isGraphSearch() ) {
// if(query.getResultsLevel() == Level.ALL_PROPERTIES)
- readPipelineBuilder.getConnection( connection );
+ if(entityType ==null )
+ readPipelineBuilder.getConnection( connection );
+ else
+ readPipelineBuilder.getConnection(connection, entityType);
//else
}
else {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c7a4b19e/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
index b8d410d..f4a4888 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
@@ -21,6 +21,7 @@ package org.apache.usergrid.corepersistence.util;
import java.util.UUID;
+import com.google.common.base.Optional;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
import org.apache.usergrid.persistence.entities.Application;
@@ -196,7 +197,7 @@ public class CpNamingUtils {
final String edgeType = getEdgeTypeFromConnectionType( connectionType );
return new SimpleSearchByEdge( sourceId, edgeType, targetId, Long.MAX_VALUE, SearchByEdgeType.Order.DESCENDING,
- null );
+ Optional.absent() );
}
[10/22] incubator-usergrid git commit: Merge branch 'two-dot-o-dev'
of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into
USERGRID-643
Posted by gr...@apache.org.
Merge branch 'two-dot-o-dev' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into USERGRID-643
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/a767bb61
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/a767bb61
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/a767bb61
Branch: refs/heads/USERGRID-608
Commit: a767bb61e68525006202a447717e6592f0dc0c8e
Parents: 20c9b35 5b1dfa1
Author: Todd Nine <tn...@apigee.com>
Authored: Thu May 14 17:19:41 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Thu May 14 17:19:41 2015 -0600
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 22 ++---
.../corepersistence/CpEntityManagerFactory.java | 8 +-
.../corepersistence/CpRelationManager.java | 90 ++++++++++----------
.../corepersistence/index/IndexServiceImpl.java | 6 +-
.../usergrid/persistence/EntityManager.java | 34 ++++----
.../persistence/MultiQueryIterator.java | 4 +-
.../apache/usergrid/persistence/PathQuery.java | 3 +-
.../usergrid/persistence/RelationManager.java | 12 +--
.../usergrid/persistence/CollectionIT.java | 2 +-
.../usergrid/persistence/CountingMutatorIT.java | 6 +-
.../persistence/EntityConnectionsIT.java | 26 +++---
.../org/apache/usergrid/persistence/GeoIT.java | 8 +-
.../PerformanceEntityRebuildIndexTest.java | 8 +-
.../persistence/query/ConnectionHelper.java | 2 +-
.../persistence/query/IteratingQueryIT.java | 2 +-
.../resources/usergrid-custom-test.properties | 2 +
.../persistence/index/impl/IndexingUtils.java | 2 +-
.../org/apache/usergrid/rest/ApiResponse.java | 16 ++++
.../apache/usergrid/rest/ApiResponseTest.java | 45 ++++++++++
.../rest/management/ImportResourceIT.java | 2 +-
.../cassandra/ManagementServiceImpl.java | 23 ++---
.../management/export/ExportServiceImpl.java | 4 +-
.../management/importer/FileImportJob.java | 5 +-
.../management/importer/ImportServiceImpl.java | 15 ++--
.../services/AbstractConnectionsService.java | 24 +++---
.../users/activities/ActivitiesService.java | 4 +-
.../users/activities/ActivitiesService.java | 4 +-
.../apache/usergrid/management/EmailFlowIT.java | 9 +-
.../management/importer/ImportCollectionIT.java | 2 +-
.../importer/ImportConnectionsTest.java | 9 +-
.../management/importer/ImportServiceIT.java | 6 +-
31 files changed, 224 insertions(+), 181 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a767bb61/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a767bb61/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
----------------------------------------------------------------------
[03/22] incubator-usergrid git commit: Merge branch 'two-dot-o-dev'
of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into
USERGRID-643
Posted by gr...@apache.org.
Merge branch 'two-dot-o-dev' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into USERGRID-643
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/b1d9ac28
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/b1d9ac28
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/b1d9ac28
Branch: refs/heads/USERGRID-608
Commit: b1d9ac283d1bf8ab17a5d9fd305b38447b5799d1
Parents: dbfff99 bc3cafb
Author: Todd Nine <tn...@apigee.com>
Authored: Tue May 12 10:43:32 2015 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Tue May 12 10:43:32 2015 -0700
----------------------------------------------------------------------
.../usergrid/corepersistence/CpRelationManager.java | 4 ++--
.../java/org/apache/usergrid/persistence/Query.java | 13 ++++---------
.../usergrid/services/AbstractCollectionService.java | 4 ++--
.../org/apache/usergrid/services/ServiceRequest.java | 2 +-
4 files changed, 9 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
[09/22] incubator-usergrid git commit: Finishes changes before tests
Posted by gr...@apache.org.
Finishes changes before tests
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/20c9b350
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/20c9b350
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/20c9b350
Branch: refs/heads/USERGRID-608
Commit: 20c9b3509cf96a6ecab1a45a2c572fd6a041e00d
Parents: cb179d3
Author: Todd Nine <tn...@apigee.com>
Authored: Thu May 14 17:19:27 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Thu May 14 17:19:27 2015 -0600
----------------------------------------------------------------------
.../asyncevents/InMemoryAsyncEventService.java | 2 -
.../index/EdgeScopeSerializer.java | 41 ++++
.../index/IndexProcessorFig.java | 6 +-
.../corepersistence/index/ReIndexService.java | 75 +++---
.../index/ReIndexServiceImpl.java | 226 +++++++++++++++----
.../pipeline/cursor/CursorSerializerUtil.java | 54 ++++-
.../pipeline/cursor/RequestCursor.java | 9 +-
.../pipeline/cursor/ResponseCursor.java | 49 ++--
.../pipeline/read/AbstractPathFilter.java | 30 ---
.../pipeline/read/CursorSeek.java | 53 +++++
.../rx/impl/AllEntityIdsObservable.java | 4 +-
.../rx/impl/AllEntityIdsObservableImpl.java | 5 +-
.../PerformanceEntityRebuildIndexTest.java | 5 +-
.../graph/serialization/EdgesObservable.java | 21 +-
.../serialization/impl/EdgesObservableImpl.java | 4 +-
15 files changed, 422 insertions(+), 162 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/20c9b350/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/InMemoryAsyncEventService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/InMemoryAsyncEventService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/InMemoryAsyncEventService.java
index 96966bf..ddcf826 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/InMemoryAsyncEventService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/InMemoryAsyncEventService.java
@@ -99,8 +99,6 @@ public class InMemoryAsyncEventService implements AsyncEventService {
@Override
public void index( final EntityIndexOperation entityIndexOperation ) {
-
-
run(eventBuilder.index( entityIndexOperation ));
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/20c9b350/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/EdgeScopeSerializer.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/EdgeScopeSerializer.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/EdgeScopeSerializer.java
new file mode 100644
index 0000000..2a6a5ac
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/EdgeScopeSerializer.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.index;
+
+
+import org.apache.usergrid.corepersistence.pipeline.cursor.AbstractCursorSerializer;
+import org.apache.usergrid.corepersistence.rx.impl.EdgeScope;
+
+
+/**
+ * Serialize our edge scope for cursors
+ */
+public class EdgeScopeSerializer extends AbstractCursorSerializer<EdgeScope> {
+
+
+ public static final EdgeScopeSerializer INSTANCE = new EdgeScopeSerializer();
+
+ @Override
+ protected Class<EdgeScope> getType() {
+ return EdgeScope.class;
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/20c9b350/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java
index fe9d3fd..8e835e2 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java
@@ -78,9 +78,9 @@ public interface IndexProcessorFig extends GuicyFig {
String getQueueImplementation();
- @Default("30000")
- @Key("elasticsearch.reindex.sample.interval")
- long getReIndexSampleInterval();
+ @Default("10000")
+ @Key("elasticsearch.reindex.flush.interval")
+ int getUpdateInterval();
@Default("false")
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/20c9b350/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java
index b25eca5..f8955dd 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java
@@ -20,22 +20,6 @@
package org.apache.usergrid.corepersistence.index;
-import java.util.UUID;
-import java.util.concurrent.Callable;
-import java.util.concurrent.FutureTask;
-
-import org.apache.usergrid.corepersistence.rx.impl.AllEntityIdsObservable;
-import org.apache.usergrid.corepersistence.rx.impl.EdgeScope;
-import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-
-import com.google.common.base.Optional;
-
-import rx.Observable;
-import rx.Observer;
-import rx.observables.ConnectableObservable;
-
-
/**
* An interface for re-indexing all entities in an application
*/
@@ -46,48 +30,75 @@ public interface ReIndexService {
* Perform an index rebuild
*
* @param indexServiceRequestBuilder The builder to build the request
- * @return
*/
- IndexResponse rebuildIndex(final IndexServiceRequestBuilder indexServiceRequestBuilder);
+ IndexResponse rebuildIndex( final IndexServiceRequestBuilder indexServiceRequestBuilder );
/**
* Generate a build for the index
- * @return
*/
IndexServiceRequestBuilder getBuilder();
+
+ /**
+ * Get the status of a job
+ * @param jobId The jobId returned during the rebuild index
+ * @return
+ */
+ IndexResponse getStatus( final String jobId );
+
+
/**
* The response when requesting a re-index operation
*/
class IndexResponse {
- final String cursor;
- final ConnectableObservable<EdgeScope> indexedEdgecount;
+ final String jobId;
+ final String status;
+ final long numberProcessed;
+ final long lastUpdated;
+
+
+ public IndexResponse( final String jobId, final String status, final long numberProcessed,
+ final long lastUpdated ) {
+ this.jobId = jobId;
+ this.status = status;
+ this.numberProcessed = numberProcessed;
+ this.lastUpdated = lastUpdated;
+ }
- public IndexResponse( final String cursor, final ConnectableObservable<EdgeScope> indexedEdgecount ) {
- this.cursor = cursor;
- this.indexedEdgecount = indexedEdgecount;
+ /**
+ * Get the jobId used to resume this operation
+ */
+ public String getJobId() {
+ return jobId;
+ }
+
+
+ /**
+ * Get the last updated time, as a long
+ * @return
+ */
+ public long getLastUpdated() {
+ return lastUpdated;
}
/**
- * Get the cursor used to resume this operation
+ * Get the number of records processed
* @return
*/
- public String getCursor() {
- return cursor;
+ public long getNumberProcessed() {
+ return numberProcessed;
}
/**
- * Return the observable of all edges to be indexed.
- *
- * Note that after subscribing "connect" will need to be called to ensure that processing begins
+ * Get the status
* @return
*/
- public ConnectableObservable<EdgeScope> getCount() {
- return indexedEdgecount;
+ public String getStatus() {
+ return status;
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/20c9b350/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
index a2fa09a..d828fc2 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
@@ -20,29 +20,32 @@
package org.apache.usergrid.corepersistence.index;
-import java.util.concurrent.TimeUnit;
+import java.util.List;
import org.apache.usergrid.corepersistence.asyncevents.AsyncEventService;
+import org.apache.usergrid.corepersistence.pipeline.cursor.CursorSerializerUtil;
+import org.apache.usergrid.corepersistence.pipeline.read.CursorSeek;
import org.apache.usergrid.corepersistence.rx.impl.AllApplicationsObservable;
import org.apache.usergrid.corepersistence.rx.impl.AllEntityIdsObservable;
import org.apache.usergrid.corepersistence.rx.impl.EdgeScope;
import org.apache.usergrid.corepersistence.util.CpNamingUtils;
-import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
-import org.apache.usergrid.persistence.core.rx.RxTaskScheduler;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.util.StringUtils;
+import org.apache.usergrid.persistence.graph.Edge;
import org.apache.usergrid.persistence.map.MapManager;
import org.apache.usergrid.persistence.map.MapManagerFactory;
import org.apache.usergrid.persistence.map.MapScope;
import org.apache.usergrid.persistence.map.impl.MapScopeImpl;
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
+import com.fasterxml.jackson.databind.JsonNode;
import com.google.common.base.Optional;
+import com.google.common.base.Preconditions;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import rx.Observable;
-import rx.observables.ConnectableObservable;
+import rx.schedulers.Schedulers;
@Singleton
@@ -51,14 +54,18 @@ public class ReIndexServiceImpl implements ReIndexService {
private static final MapScope RESUME_MAP_SCOPE =
new MapScopeImpl( CpNamingUtils.getManagementApplicationId(), "reindexresume" );
- //Keep cursors to resume re-index for 1 day. This is far beyond it's useful real world implications anyway.
+ //Keep cursors to resume re-index for 10 days. This is far beyond it's useful real world implications anyway.
private static final int INDEX_TTL = 60 * 60 * 24 * 10;
+ private static final String MAP_CURSOR_KEY = "cursor";
+ private static final String MAP_COUNT_KEY = "count";
+ private static final String MAP_STATUS_KEY = "status";
+ private static final String MAP_UPDATED_KEY = "lastUpdated";
+
private final AllApplicationsObservable allApplicationsObservable;
private final AllEntityIdsObservable allEntityIdsObservable;
private final IndexProcessorFig indexProcessorFig;
- private final RxTaskScheduler rxTaskScheduler;
private final MapManager mapManager;
private final AsyncEventService indexService;
@@ -66,69 +73,61 @@ public class ReIndexServiceImpl implements ReIndexService {
@Inject
public ReIndexServiceImpl( final AllEntityIdsObservable allEntityIdsObservable,
final MapManagerFactory mapManagerFactory,
- final AllApplicationsObservable allApplicationsObservable, final IndexProcessorFig indexProcessorFig,
- final RxTaskScheduler rxTaskScheduler, final AsyncEventService indexService ) {
+ final AllApplicationsObservable allApplicationsObservable,
+ final IndexProcessorFig indexProcessorFig, final AsyncEventService indexService ) {
this.allEntityIdsObservable = allEntityIdsObservable;
this.allApplicationsObservable = allApplicationsObservable;
this.indexProcessorFig = indexProcessorFig;
- this.rxTaskScheduler = rxTaskScheduler;
this.indexService = indexService;
this.mapManager = mapManagerFactory.createMapManager( RESUME_MAP_SCOPE );
}
-
-
-
@Override
public IndexResponse rebuildIndex( final IndexServiceRequestBuilder indexServiceRequestBuilder ) {
- //load our last emitted Scope if a cursor is present
- if ( indexServiceRequestBuilder.getCursor().isPresent() ) {
- throw new UnsupportedOperationException( "Build this" );
- }
+ //load our last emitted Scope if a cursor is present
+ final Optional<EdgeScope> cursor = parseCursor( indexServiceRequestBuilder.getCursor() );
+
+
+ final CursorSeek<Edge> cursorSeek = getResumeEdge( cursor );
final Optional<ApplicationScope> appId = indexServiceRequestBuilder.getApplicationScope();
- final Observable<ApplicationScope> applicationScopes = appId.isPresent()? Observable.just( appId.get() ) : allApplicationsObservable.getData();
+ Preconditions.checkArgument( cursor.isPresent() && appId.isPresent(),
+ "You cannot specify an app id and a cursor. When resuming with cursor you must omit the appid" );
+
+ final Observable<ApplicationScope> applicationScopes = getApplications( cursor, appId );
- final String newCursor = StringUtils.sanitizeUUID( UUIDGenerator.newTimeUUID() );
+ final String jobId = StringUtils.sanitizeUUID( UUIDGenerator.newTimeUUID() );
final long modifiedSince = indexServiceRequestBuilder.getUpdateTimestamp().or( Long.MIN_VALUE );
//create an observable that loads each entity and indexes it, start it running with publish
- final ConnectableObservable<EdgeScope> runningReIndex =
- allEntityIdsObservable.getEdgesToEntities( applicationScopes,
- indexServiceRequestBuilder.getCollectionName() )
-
- //for each edge, create our scope and index on it
- .doOnNext( edge -> indexService.index(
- new EntityIndexOperation( edge.getApplicationScope(), edge.getEdge().getTargetNode(),
- modifiedSince ) ) ).publish();
+ final Observable<EdgeScope> runningReIndex = allEntityIdsObservable.getEdgesToEntities( applicationScopes,
+ indexServiceRequestBuilder.getCollectionName(), cursorSeek.getSeekValue() )
+ //for each edge, create our scope and index on it
+ .doOnNext( edge -> indexService.index(
+ new EntityIndexOperation( edge.getApplicationScope(), edge.getEdge().getTargetNode(),
+ modifiedSince ) ) );
//start our sampler and state persistence
//take a sample every sample interval to allow us to resume state with minimal loss
- runningReIndex.sample( indexProcessorFig.getReIndexSampleInterval(), TimeUnit.MILLISECONDS,
- rxTaskScheduler.getAsyncIOScheduler() )
- .doOnNext( edge -> {
-
-// final String serializedState = SerializableMapper.asString( edge );
-//
-// mapManager.putString( newCursor, serializedState, INDEX_TTL );
- } ).subscribe();
+ runningReIndex.buffer( indexProcessorFig.getUpdateInterval() )
+ //create our flushing collector and flush the edge scopes to it
+ .collect( () -> new FlushingCollector( jobId ),
+ ( ( flushingCollector, edgeScopes ) -> flushingCollector.flushBuffer( edgeScopes ) ) ).doOnNext( flushingCollector-> flushingCollector.complete() )
+ //subscribe on our I/O scheduler and run the task
+ .subscribeOn( Schedulers.io() ).subscribe();
- //start pushing to both
- runningReIndex.connect();
-
-
- return new IndexResponse( newCursor, runningReIndex );
+ return new IndexResponse( jobId, "Started", 0, 0 );
}
@@ -136,6 +135,155 @@ public class ReIndexServiceImpl implements ReIndexService {
public IndexServiceRequestBuilder getBuilder() {
return new IndexServiceRequestBuilderImpl();
}
+
+
+ @Override
+ public IndexResponse getStatus( final String jobId ) {
+ Preconditions.checkNotNull( jobId, "jobId must not be null" );
+ return getIndexResponse( jobId );
+ }
+
+
+ /**
+ * Simple collector that counts state, then flushed every time a buffer is provided. Writes final state when complete
+ */
+ private class FlushingCollector {
+
+ private final String jobId;
+ private long count;
+
+
+ private FlushingCollector( final String jobId ) {
+ this.jobId = jobId;
+ }
+
+
+ public void flushBuffer( final List<EdgeScope> buffer ) {
+ count += buffer.size();
+
+ //write our cursor state
+ if ( buffer.size() > 0 ) {
+ writeCursorState( jobId, buffer.get( buffer.size() - 1 ) );
+ }
+
+ writeStateMeta( jobId, "InProgress", count, System.currentTimeMillis() );
+ }
+
+ public void complete(){
+ writeStateMeta( jobId, "Complete", count, System.currentTimeMillis() );
+ }
+ }
+
+
+ /**
+ * Get the resume edge scope
+ *
+ * @param edgeScope The optional edge scope from the cursor
+ */
+ private CursorSeek<Edge> getResumeEdge( final Optional<EdgeScope> edgeScope ) {
+
+
+ if ( edgeScope.isPresent() ) {
+ return new CursorSeek<>( Optional.of( edgeScope.get().getEdge() ) );
+ }
+
+ return new CursorSeek<>( Optional.absent() );
+ }
+
+
+ /**
+ * Generate an observable for our appliation scope
+ */
+ private Observable<ApplicationScope> getApplications( final Optional<EdgeScope> cursor,
+ final Optional<ApplicationScope> appId ) {
+ //cursor is present use it and skip until we hit that app
+ if ( cursor.isPresent() ) {
+
+ final EdgeScope cursorValue = cursor.get();
+ //we have a cursor and an application scope that was used.
+ return allApplicationsObservable.getData().skipWhile(
+ applicationScope -> !cursorValue.getApplicationScope().equals( applicationScope ) );
+ }
+ //this is intentional. If
+ else if ( appId.isPresent() ) {
+ return Observable.just( appId.get() );
+ }
+
+ return allApplicationsObservable.getData();
+ }
+
+
+ /**
+ * Swap our cursor for an optional edgescope
+ */
+ private Optional<EdgeScope> parseCursor( final Optional<String> cursor ) {
+
+ if ( !cursor.isPresent() ) {
+ return Optional.absent();
+ }
+
+ //get our cursor
+ final String persistedCursor = mapManager.getString( cursor.get() );
+
+ if ( persistedCursor == null ) {
+ return Optional.absent();
+ }
+
+ final JsonNode node = CursorSerializerUtil.fromString( persistedCursor );
+
+ final EdgeScope edgeScope = EdgeScopeSerializer.INSTANCE.fromJsonNode( node, CursorSerializerUtil.getMapper() );
+
+ return Optional.of( edgeScope );
+ }
+
+
+ /**
+ * Write the cursor state to the map in cassandra
+ */
+ private void writeCursorState( final String jobId, final EdgeScope edge ) {
+
+ final JsonNode node = EdgeScopeSerializer.INSTANCE.toNode( CursorSerializerUtil.getMapper(), edge );
+
+ final String serializedState = CursorSerializerUtil.asString( node );
+
+ mapManager.putString( jobId + MAP_CURSOR_KEY, serializedState, INDEX_TTL );
+ }
+
+
+ /**
+ * Write our state meta data into cassandra so everyone can see it
+ * @param jobId
+ * @param status
+ * @param processedCount
+ * @param lastUpdated
+ */
+ private void writeStateMeta( final String jobId, final String status, final long processedCount,
+ final long lastUpdated ) {
+
+ mapManager.putString( jobId + MAP_STATUS_KEY, status );
+ mapManager.putLong( jobId + MAP_COUNT_KEY, processedCount );
+ mapManager.putLong( jobId + MAP_UPDATED_KEY, lastUpdated );
+ }
+
+
+ /**
+ * Get the index response from the jobId
+ * @param jobId
+ * @return
+ */
+ private IndexResponse getIndexResponse( final String jobId ) {
+
+ final String status = mapManager.getString( jobId+MAP_STATUS_KEY );
+
+ if(status == null){
+ throw new IllegalArgumentException( "Could not find a job with id " + jobId );
+ }
+
+ final long processedCount = mapManager.getLong( jobId + MAP_COUNT_KEY );
+ final long lastUpdated = mapManager.getLong( jobId + MAP_COUNT_KEY );
+
+ return new IndexResponse( jobId, status, processedCount, lastUpdated );
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/20c9b350/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/cursor/CursorSerializerUtil.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/cursor/CursorSerializerUtil.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/cursor/CursorSerializerUtil.java
index fea0364..7acdd00 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/cursor/CursorSerializerUtil.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/cursor/CursorSerializerUtil.java
@@ -20,10 +20,15 @@
package org.apache.usergrid.corepersistence.pipeline.cursor;
-import com.fasterxml.jackson.core.Base64Variant;
-import com.fasterxml.jackson.core.Base64Variants;
+import java.io.IOException;
+import java.util.Base64;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
import com.fasterxml.jackson.dataformat.smile.SmileFactory;
+import com.google.common.base.Preconditions;
/**
@@ -35,9 +40,54 @@ public class CursorSerializerUtil {
private static final ObjectMapper MAPPER = new ObjectMapper( SMILE_FACTORY );
+ /**
+ * Aritrary number, just meant to keep us from having a DOS issue
+ */
+ private static final int MAX_SIZE = 1024;
+
public static ObjectMapper getMapper() {
return MAPPER;
}
+
+ /**
+ * Turn the json node in to a base64 encoded SMILE binary
+ */
+ public static String asString( final JsonNode node ) {
+ final byte[] output;
+ try {
+ output = MAPPER.writeValueAsBytes( node );
+ }
+ catch ( JsonProcessingException e ) {
+ throw new RuntimeException( "Unable to create output from json node " + node );
+ }
+
+ //generate a base64 url save string
+ final String value = Base64.getUrlEncoder().encodeToString( output );
+
+ return value;
+ }
+
+
+ /**
+ * Parse the base64 encoded binary string
+ */
+ public static JsonNode fromString( final String base64EncodedJson ) {
+
+ Preconditions.checkArgument( base64EncodedJson.length() <= MAX_SIZE,
+ "Your cursor must be less than " + MAX_SIZE + " chars in length" );
+
+ final byte[] data = Base64.getUrlDecoder().decode( base64EncodedJson );
+
+ JsonNode jsonNode;
+ try {
+ jsonNode = MAPPER.readTree( data );
+ }
+ catch ( IOException e ) {
+ throw new RuntimeException( "Unable to parse json node from string " + base64EncodedJson );
+ }
+
+ return jsonNode;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/20c9b350/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/cursor/RequestCursor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/cursor/RequestCursor.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/cursor/RequestCursor.java
index 870edbb..dc6ae71 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/cursor/RequestCursor.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/cursor/RequestCursor.java
@@ -37,10 +37,6 @@ import com.google.common.base.Preconditions;
*/
public class RequestCursor {
- /**
- * Aritrary number, just meant to keep us from having a DOS issue
- */
- private static final int MAX_SIZE = 1024;
private static final int MAX_CURSOR_COUNT = 100;
@@ -83,11 +79,8 @@ public class RequestCursor {
try {
- Preconditions.checkArgument( cursor.length() <= MAX_SIZE, "Your cursor must be less than " + MAX_SIZE + " chars in length");
-
- final byte[] data = Base64.getUrlDecoder().decode( cursor );
- JsonNode jsonNode = MAPPER.readTree( data );
+ JsonNode jsonNode = CursorSerializerUtil.fromString( cursor );
Preconditions
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/20c9b350/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/cursor/ResponseCursor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/cursor/ResponseCursor.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/cursor/ResponseCursor.java
index dbd8b88..dc4bf39 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/cursor/ResponseCursor.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/cursor/ResponseCursor.java
@@ -20,11 +20,8 @@
package org.apache.usergrid.corepersistence.pipeline.cursor;
-import java.util.Base64;
-
import org.apache.usergrid.corepersistence.pipeline.read.EdgePath;
-import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
@@ -53,8 +50,8 @@ public class ResponseCursor {
/**
- * Lazyily encoded deliberately. If the user doesn't care about a cursor and is using streams, we dont' want to take the
- * time to calculate it
+ * Lazyily encoded deliberately. If the user doesn't care about a cursor and is using streams, we dont' want to
+ * take the time to calculate it
*/
public Optional<String> encodeAsString() {
@@ -68,42 +65,34 @@ public class ResponseCursor {
return encodedValue;
}
+ //no edge path, short circuit
- try {
-
- //no edge path, short circuit
-
- final ObjectNode map = MAPPER.createObjectNode();
+ final ObjectNode map = MAPPER.createObjectNode();
- Optional<EdgePath> current = edgePath;
+ Optional<EdgePath> current = edgePath;
- //traverse each edge and add them to our json
- do {
+ //traverse each edge and add them to our json
+ do {
- final EdgePath edgePath = current.get();
- final Object cursorValue = edgePath.getCursorValue();
- final CursorSerializer serializer = edgePath.getSerializer();
- final int filterId = edgePath.getFilterId();
+ final EdgePath edgePath = current.get();
+ final Object cursorValue = edgePath.getCursorValue();
+ final CursorSerializer serializer = edgePath.getSerializer();
+ final int filterId = edgePath.getFilterId();
- final JsonNode serialized = serializer.toNode( MAPPER, cursorValue );
- map.put( String.valueOf( filterId ), serialized );
+ final JsonNode serialized = serializer.toNode( MAPPER, cursorValue );
+ map.put( String.valueOf( filterId ), serialized );
- current = current.get().getPrevious();
- }
- while ( current.isPresent() );
+ current = current.get().getPrevious();
+ }
+ while ( current.isPresent() );
- final byte[] output = MAPPER.writeValueAsBytes( map );
+ //generate a base64 url save string
+ final String value = CursorSerializerUtil.asString( map );
- //generate a base64 url save string
- final String value = Base64.getUrlEncoder().encodeToString( output );
+ encodedValue = Optional.of( value );
- encodedValue = Optional.of( value );
- }
- catch ( JsonProcessingException e ) {
- throw new CursorParseException( "Unable to serialize cursor", e );
- }
return encodedValue;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/20c9b350/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/AbstractPathFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/AbstractPathFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/AbstractPathFilter.java
index c68dc4a..0f9ac9b 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/AbstractPathFilter.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/AbstractPathFilter.java
@@ -76,34 +76,4 @@ public abstract class AbstractPathFilter<T, R, C extends Serializable> extends A
* Return the class to be used when parsing the cursor
*/
protected abstract CursorSerializer<C> getCursorSerializer();
-
-
- /**
- * An internal class that holds a mutable state. When resuming, we only ever honor the seek value on the first call. Afterwards, we will seek from the beginning on newly emitted values.
- * Calling get will return the first value to seek, or absent if not specified. Subsequent calls will return absent. Callers should treat the results as seek values for each operation
- */
- protected static class CursorSeek<C> {
-
- private Optional<C> seek;
-
- private CursorSeek(final Optional<C> cursorValue){
- seek = cursorValue;
- }
-
-
- /**
- * Get the seek value to use when searching
- * @return
- */
- public Optional<C> getSeekValue(){
- final Optional<C> toReturn = seek;
-
- seek = Optional.absent();
-
- return toReturn;
- }
-
-
-
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/20c9b350/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/CursorSeek.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/CursorSeek.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/CursorSeek.java
new file mode 100644
index 0000000..b803658
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/CursorSeek.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline.read;
+
+
+import com.google.common.base.Optional;
+
+
+/**
+ * An internal class that holds a mutable state. When resuming, we only ever honor the seek value on the first call. Afterwards, we will seek from the beginning on newly emitted values.
+ * Calling get will return the first value to seek, or absent if not specified. Subsequent calls will return absent. Callers should treat the results as seek values for each operation
+ */
+public class CursorSeek<C> {
+
+ private Optional<C> seek;
+
+ public CursorSeek( final Optional<C> cursorValue ){
+ seek = cursorValue;
+ }
+
+
+ /**
+ * Get the seek value to use when searching
+ * @return
+ */
+ public Optional<C> getSeekValue(){
+ final Optional<C> toReturn = seek;
+
+ seek = Optional.absent();
+
+ return toReturn;
+ }
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/20c9b350/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntityIdsObservable.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntityIdsObservable.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntityIdsObservable.java
index aada240..9070609 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntityIdsObservable.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntityIdsObservable.java
@@ -24,6 +24,7 @@ import com.google.common.base.Optional;
import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.graph.Edge;
import rx.Observable;
@@ -44,8 +45,9 @@ public interface AllEntityIdsObservable {
* Get all edges that represent edges to entities in the system
* @param appScopes
* @param edgeType The edge type to use (if specified)
+ * @param lastEdge The edge to resume processing from
* @return
*/
- Observable<EdgeScope> getEdgesToEntities(final Observable<ApplicationScope> appScopes, final Optional<String> edgeType);
+ Observable<EdgeScope> getEdgesToEntities(final Observable<ApplicationScope> appScopes, final Optional<String> edgeType, final Optional<Edge> lastEdge);
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/20c9b350/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntityIdsObservableImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntityIdsObservableImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntityIdsObservableImpl.java
index 6a95e7b..0420a32 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntityIdsObservableImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntityIdsObservableImpl.java
@@ -28,6 +28,7 @@ import com.google.inject.Singleton;
import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.graph.Edge;
import org.apache.usergrid.persistence.graph.GraphManager;
import org.apache.usergrid.persistence.graph.GraphManagerFactory;
import org.apache.usergrid.persistence.graph.serialization.EdgesObservable;
@@ -81,12 +82,12 @@ public class AllEntityIdsObservableImpl implements AllEntityIdsObservable {
@Override
- public Observable<EdgeScope> getEdgesToEntities( final Observable<ApplicationScope> appScopes, final Optional<String> edgeType) {
+ public Observable<EdgeScope> getEdgesToEntities( final Observable<ApplicationScope> appScopes, final Optional<String> edgeType, final Optional<Edge> lastEdge) {
return appScopes.flatMap( applicationScope -> {
final GraphManager gm = graphManagerFactory.createEdgeManager( applicationScope );
- return edgesObservable.edgesFromSourceDescending( gm, applicationScope.getApplication(), edgeType )
+ return edgesObservable.edgesFromSourceDescending( gm, applicationScope.getApplication(), edgeType, lastEdge )
.map( edge -> new EdgeScope(applicationScope, edge ));
} );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/20c9b350/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java b/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
index a17c925..cb9919f 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
@@ -27,6 +27,7 @@ import java.util.concurrent.TimeUnit;
import com.google.common.base.Optional;
import org.apache.commons.lang.RandomStringUtils;
+import org.apache.usergrid.corepersistence.index.IndexServiceRequestBuilder;
import org.apache.usergrid.corepersistence.index.ReIndexService;
import org.apache.usergrid.persistence.index.ApplicationEntityIndex;
import org.junit.After;
@@ -196,7 +197,9 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT {
try {
- reIndexService.rebuildIndex( Optional.of( em.getApplicationId()), Optional.<String>of("catherders"), Optional.absent(), Optional.absent() );
+ final IndexServiceRequestBuilder builder = reIndexService.getBuilder().withApplicationId( em.getApplicationId() ).withCollection( "catherders" );
+
+ reIndexService.rebuildIndex(builder );
reporter.report();
registry.remove( meterName );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/20c9b350/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/EdgesObservable.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/EdgesObservable.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/EdgesObservable.java
index 964e13d..78a1d4b 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/EdgesObservable.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/EdgesObservable.java
@@ -42,16 +42,6 @@ public interface EdgesObservable {
/**
- * Return an observable of all edges from a source node. Ordered ascending, from the startTimestamp if specified
- * @param gm
- * @param sourceNode
- * @param edgeType The edge type if specified. Otherwise all types will be used
- * @return
- */
- Observable<Edge> edgesFromSourceDescending( final GraphManager gm, final Id sourceNode,
- final Optional<String> edgeType );
-
- /**
* Get all edges from the source node with the target type
* @param gm
* @param sourceNode
@@ -67,4 +57,15 @@ public interface EdgesObservable {
* @return
*/
Observable<Edge> edgesToTarget(final GraphManager gm, final Id targetNode);
+
+ /**
+ * Return an observable of all edges from a source node. Ordered ascending, from the startTimestamp if specified
+ * @param gm
+ * @param sourceNode
+ * @param edgeType The edge type if specified. Otherwise all types will be used
+ * @param resume The edge to start seeking after. Otherwise starts at the most recent
+ * @return
+ */
+ Observable<Edge> edgesFromSourceDescending( final GraphManager gm, final Id sourceNode,
+ final Optional<String> edgeType, final Optional<Edge> resume );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/20c9b350/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgesObservableImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgesObservableImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgesObservableImpl.java
index 7240798..18274ac 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgesObservableImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgesObservableImpl.java
@@ -72,7 +72,7 @@ public class EdgesObservableImpl implements EdgesObservable {
@Override
public Observable<Edge> edgesFromSourceDescending( final GraphManager gm, final Id sourceNode,
- final Optional<String> edgeTypeInput ) {
+ final Optional<String> edgeTypeInput, final Optional<Edge> resume ) {
@@ -86,7 +86,7 @@ public class EdgesObservableImpl implements EdgesObservable {
return gm.loadEdgesFromSource(
new SimpleSearchByEdgeType( sourceNode, edgeType, Long.MAX_VALUE, SearchByEdgeType.Order.DESCENDING,
- Optional.<Edge>absent() ) );
+ resume ) );
} );
}
[22/22] incubator-usergrid git commit: Merge branch 'two-dot-o-dev'
of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into
USERGRID-608
Posted by gr...@apache.org.
Merge branch 'two-dot-o-dev' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into USERGRID-608
# By Todd Nine (7) and others
# Via Shawn Feldman (2) and others
* 'two-dot-o-dev' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid:
[USERGRID-642] Remove password properties from user objects in portal
Addresses some of George's comments
Fixes exception on missing keyspace during bootstrap
Updates REST api calls. POST is used to create a re-index job, PUT is used to resume.
test fixes
fix serviceinvocation it
fix serviceinvocation it
Change usergrid SDK logging to be set via config.
Fixes bugs and cleans up tests
Finishes changes before tests
fix connections
rename connecting to source and connected to target
rename vars
Refactored observable methods to correct name
Update of Apis and tests
[USERGRID-572] - ignore sensitive params from QP in response params
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/3f8899df
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/3f8899df
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/3f8899df
Branch: refs/heads/USERGRID-608
Commit: 3f8899dfa10a22a4b67f0099d86c19ed22ba6c1e
Parents: aa8b20d 53563e8
Author: GERey <gr...@apigee.com>
Authored: Tue May 26 10:55:31 2015 -0700
Committer: GERey <gr...@apigee.com>
Committed: Tue May 26 10:55:31 2015 -0700
----------------------------------------------------------------------
portal/config.js | 4 +-
portal/js/global/ug-service.js | 4 +-
portal/js/users/users-controller.js | 19 +-
.../usergrid/corepersistence/CoreModule.java | 4 +
.../corepersistence/CpEntityManager.java | 24 +-
.../corepersistence/CpEntityManagerFactory.java | 22 +-
.../corepersistence/CpRelationManager.java | 123 +++----
.../asyncevents/EventBuilder.java | 6 +-
.../asyncevents/EventBuilderImpl.java | 18 +-
.../asyncevents/InMemoryAsyncEventService.java | 9 +-
.../asyncevents/SQSAsyncEventService.java | 9 +-
.../index/EdgeScopeSerializer.java | 41 +++
.../index/EntityIndexOperation.java | 46 +++
.../index/IndexProcessorFig.java | 6 +-
.../corepersistence/index/IndexServiceImpl.java | 9 +-
.../corepersistence/index/ReIndexAction.java | 6 +-
.../index/ReIndexRequestBuilder.java | 86 +++++
.../index/ReIndexRequestBuilderImpl.java | 122 +++++++
.../corepersistence/index/ReIndexService.java | 89 +++--
.../index/ReIndexServiceImpl.java | 251 +++++++++++--
.../cursor/AbstractCursorSerializer.java | 2 +-
.../pipeline/cursor/CursorSerializerUtil.java | 54 ++-
.../pipeline/cursor/RequestCursor.java | 9 +-
.../pipeline/cursor/ResponseCursor.java | 49 +--
.../pipeline/read/AbstractPathFilter.java | 30 --
.../pipeline/read/CursorSeek.java | 53 +++
.../rx/impl/AllEntityIdsObservable.java | 5 +-
.../rx/impl/AllEntityIdsObservableImpl.java | 6 +-
.../corepersistence/util/CpNamingUtils.java | 3 +-
.../util/SerializableMapper.java | 91 -----
.../usergrid/persistence/EntityManager.java | 34 +-
.../persistence/EntityManagerFactory.java | 2 -
.../persistence/MultiQueryIterator.java | 4 +-
.../apache/usergrid/persistence/PathQuery.java | 3 +-
.../usergrid/persistence/RelationManager.java | 14 +-
.../rx/EdgesToTargetObservableIT.java | 4 +-
.../usergrid/persistence/CollectionIT.java | 2 +-
.../usergrid/persistence/CountingMutatorIT.java | 6 +-
.../persistence/EntityConnectionsIT.java | 26 +-
.../org/apache/usergrid/persistence/GeoIT.java | 8 +-
.../PerformanceEntityRebuildIndexTest.java | 350 ++++++++-----------
.../persistence/query/ConnectionHelper.java | 2 +-
.../persistence/query/IteratingQueryIT.java | 2 +-
.../data/MigrationInfoSerializationImpl.java | 4 +-
.../migration/schema/MigrationManagerImpl.java | 14 +-
.../core/migration/util/AstayanxUtils.java | 49 +++
.../graph/serialization/EdgesObservable.java | 24 +-
.../serialization/impl/EdgesObservableImpl.java | 12 +-
.../impl/TargetIdObservableImpl.java | 2 +-
.../impl/migration/EdgeDataMigrationImpl.java | 2 +-
.../persistence/index/impl/IndexingUtils.java | 2 +
stack/pom.xml | 13 +
.../org/apache/usergrid/rest/ApiResponse.java | 16 +
.../org/apache/usergrid/rest/IndexResource.java | 342 +++++++++---------
.../main/resources/usergrid-rest-context.xml | 3 -
.../resources/usergrid-rest-deploy-context.xml | 1 -
.../apache/usergrid/rest/ApiResponseTest.java | 45 +++
.../rest/management/ImportResourceIT.java | 2 +-
.../cassandra/ManagementServiceImpl.java | 15 +-
.../management/export/ExportServiceImpl.java | 4 +-
.../management/importer/FileImportJob.java | 5 +-
.../management/importer/ImportServiceImpl.java | 15 +-
.../services/AbstractConnectionsService.java | 24 +-
.../users/activities/ActivitiesService.java | 4 +-
.../users/activities/ActivitiesService.java | 4 +-
.../usergrid/management/OrganizationIT.java | 8 +-
.../management/importer/ImportCollectionIT.java | 2 +-
.../importer/ImportConnectionsTest.java | 9 +-
.../management/importer/ImportServiceIT.java | 6 +-
.../resources/usergrid-custom-test.properties | 1 +
70 files changed, 1392 insertions(+), 893 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3f8899df/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3f8899df/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3f8899df/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3f8899df/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/InMemoryAsyncEventService.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3f8899df/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
----------------------------------------------------------------------
[15/22] incubator-usergrid git commit: test fixes
Posted by gr...@apache.org.
test fixes
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/4116ce5b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/4116ce5b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/4116ce5b
Branch: refs/heads/USERGRID-608
Commit: 4116ce5b40bfde6d4a37ae366595ad5d3444eb2c
Parents: b8a95e9
Author: Shawn Feldman <sf...@apache.org>
Authored: Fri May 15 13:25:45 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Fri May 15 13:25:45 2015 -0600
----------------------------------------------------------------------
.../corepersistence/asyncevents/InMemoryAsyncEventService.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4116ce5b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/InMemoryAsyncEventService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/InMemoryAsyncEventService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/InMemoryAsyncEventService.java
index 6faa695..2620056 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/InMemoryAsyncEventService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/InMemoryAsyncEventService.java
@@ -110,7 +110,7 @@ public class InMemoryAsyncEventService implements AsyncEventService {
observable.subscribeOn( rxTaskScheduler.getAsyncIOScheduler() ).subscribe();
}
else {
- observable.toBlocking().last();
+ observable.toBlocking().lastOrDefault(null);
}
}
}
[19/22] incubator-usergrid git commit: Addresses some of George's
comments
Posted by gr...@apache.org.
Addresses some of George's comments
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/53aa87ca
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/53aa87ca
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/53aa87ca
Branch: refs/heads/USERGRID-608
Commit: 53aa87cad41b6d7320c24af4a5e71061fe2a4dd3
Parents: 8fde7c5
Author: Todd Nine <tn...@apigee.com>
Authored: Wed May 20 15:04:09 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed May 20 15:04:09 2015 -0600
----------------------------------------------------------------------
.../usergrid/persistence/PerformanceEntityRebuildIndexTest.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/53aa87ca/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java b/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
index 06eb060..318a378 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
@@ -314,7 +314,7 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT {
}
}
catch ( IllegalArgumentException iae ) {
- //swallow
+ //swallow. Thrown if our job can't be found. I.E hasn't updated yet
}