You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sf...@apache.org on 2014/10/02 19:46:19 UTC
[1/6] git commit: Fix to prevent infinite recursion in IndexRebuild.
Repository: incubator-usergrid
Updated Branches:
refs/heads/map 9a8bed510 -> 587cb5a73
Fix to prevent infinite recursion in IndexRebuild.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/e939b05c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/e939b05c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/e939b05c
Branch: refs/heads/map
Commit: e939b05c5e21b25a95905dec84d54a955d00f756
Parents: b2b8886
Author: Dave Johnson <dm...@apigee.com>
Authored: Thu Oct 2 10:14:23 2014 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Thu Oct 2 10:14:23 2014 -0400
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 26 ++++++++++++++------
1 file changed, 19 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e939b05c/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 38453e1..f7df6c0 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -33,6 +33,7 @@ import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.Stack;
import java.util.TreeMap;
import java.util.TreeSet;
import java.util.UUID;
@@ -2857,7 +2858,11 @@ public class CpEntityManager implements EntityManager {
* Completely reindex the application associated with this EntityManager.
*/
public void reindex( EntityManagerFactory.ProgressObserver po ) throws Exception {
- indexEntityConnectionsAndCollections( getApplication(), po );
+
+ Stack stack = new Stack();
+ stack.push( getApplication() );
+
+ indexEntityConnectionsAndCollections( getApplication(), po, stack );
}
@@ -2865,8 +2870,8 @@ public class CpEntityManager implements EntityManager {
* Recursively index (or reindex) all of the collections and connections of a
* specified entity, and all of the collected and connected entities as well.
*/
- private void indexEntityConnectionsAndCollections(
- final EntityRef entity, final EntityManagerFactory.ProgressObserver po ) {
+ private void indexEntityConnectionsAndCollections( final EntityRef entity,
+ final EntityManagerFactory.ProgressObserver po, final Stack stack ) {
final GraphManager gm = managerCache.getGraphManager(applicationScope);
@@ -2961,8 +2966,12 @@ public class CpEntityManager implements EntityManager {
po.onProgress( entity, ref, edge.getType());
// recursion
- indexEntityConnectionsAndCollections( new SimpleEntityRef(
- memberEntity.getId().getType(), memberEntity.getId().getUuid()),po);
+ if ( !stack.contains( ref )) {
+ stack.push( ref );
+ indexEntityConnectionsAndCollections( ref, po, stack );
+ stack.pop();
+ }
+
} else if ( isConnectionEdgeType( edge.getType() )) {
@@ -2998,8 +3007,11 @@ public class CpEntityManager implements EntityManager {
po.onProgress( entity, ref, edge.getType());
// recursion
- indexEntityConnectionsAndCollections( new SimpleEntityRef(
- targetEntity.getId().getType(), targetEntity.getId().getUuid()),po);
+ if ( !stack.contains( ref )) {
+ stack.push( ref );
+ indexEntityConnectionsAndCollections( ref, po, stack );
+ stack.pop();
+ }
}
}
[2/6] git commit: Merge branch 'two-dot-o' of
https://git-wip-us.apache.org/repos/asf/incubator-usergrid into two-dot-o
Posted by sf...@apache.org.
Merge branch 'two-dot-o' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into two-dot-o
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/a6665f11
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/a6665f11
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/a6665f11
Branch: refs/heads/map
Commit: a6665f113d2955a5d4cdc5dd0d3eddccc8967b50
Parents: e939b05 38885fa
Author: Dave Johnson <dm...@apigee.com>
Authored: Thu Oct 2 10:14:35 2014 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Thu Oct 2 10:14:35 2014 -0400
----------------------------------------------------------------------
.../CpEntityIndexDeleteListener.java | 11 +-
.../corepersistence/CpEntityManager.java | 247 +++-----
.../corepersistence/CpEntityManagerFactory.java | 29 +-
.../corepersistence/CpManagerCache.java | 12 +-
.../usergrid/corepersistence/CpNamingUtils.java | 106 ++++
.../corepersistence/CpRelationManager.java | 229 +++----
.../CpEntityIndexDeleteListenerTest.java | 18 +-
.../usergrid/persistence/CollectionIT.java | 3 -
.../PerformanceEntityRebuildIndexTest.java | 7 +-
.../collection/impl/CollectionScopeImpl.java | 6 +-
.../core/scope/ApplicationScopeImpl.java | 6 +-
.../serialization/EdgeSerializationTest.java | 3 +
.../usergrid/persistence/index/EntityIndex.java | 38 +-
.../persistence/index/EntityIndexBatch.java | 71 +++
.../persistence/index/EntityIndexFactory.java | 4 +-
.../usergrid/persistence/index/IndexScope.java | 2 +-
.../index/impl/EsEntityIndexBatchImpl.java | 396 ++++++++++++
.../index/impl/EsEntityIndexImpl.java | 628 ++++---------------
.../persistence/index/impl/EsQueryVistor.java | 35 +-
.../persistence/index/impl/IndexScopeImpl.java | 8 +-
.../persistence/index/impl/IndexingUtils.java | 107 ++++
.../usergrid/persistence/index/query/Query.java | 2 +-
.../index/query/tree/AndOperand.java | 2 +-
.../persistence/index/query/tree/Property.java | 2 +-
.../index/query/tree/StringLiteral.java | 2 +-
.../index/utils/IndexValidationUtils.java | 3 +-
.../index/impl/CorePerformanceIT.java | 97 +--
.../impl/EntityConnectionIndexImplTest.java | 16 +-
.../persistence/index/impl/EntityIndexTest.java | 144 +++--
.../apache/usergrid/rest/PartialUpdateTest.java | 120 ++--
30 files changed, 1326 insertions(+), 1028 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a6665f11/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index f7df6c0,d729c97..8d3940d
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@@ -2966,17 -2888,13 +2893,17 @@@ public class CpEntityManager implement
po.onProgress( entity, ref, edge.getType());
// recursion
- indexEntityConnectionsAndCollections( new SimpleEntityRef(
- memberEntity.getId().getType(), memberEntity.getId().getUuid()),po);
+ if ( !stack.contains( ref )) {
+ stack.push( ref );
+ indexEntityConnectionsAndCollections( ref, po, stack );
+ stack.pop();
+ }
+
- } else if ( isConnectionEdgeType( edge.getType() )) {
+ } else if ( CpNamingUtils.isConnectionEdgeType( edge.getType() )) {
- String connType = getConnectionType(edgeType);
- String targetEntityType = getConnectedEntityType(edgeType);
+ String connType = CpNamingUtils.getConnectionType( edgeType );
+ String targetEntityType = edge.getTargetNode().getType();
String sourceEntityType = entity.getType();
CollectionScope sourceScope = new CollectionScopeImpl(
[5/6] git commit: Log and ignore suspected empty collections and
connections.
Posted by sf...@apache.org.
Log and ignore suspected empty collections and connections.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/cff6e1cb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/cff6e1cb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/cff6e1cb
Branch: refs/heads/map
Commit: cff6e1cbf632900a6d0610dff1fa8ae6145bffe8
Parents: 878d634
Author: Dave Johnson <dm...@apigee.com>
Authored: Thu Oct 2 12:41:34 2014 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Thu Oct 2 12:41:34 2014 -0400
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 80 +++++++++-----------
1 file changed, 36 insertions(+), 44 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cff6e1cb/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index f618a80..d7f3a3b 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -2847,17 +2847,15 @@ public class CpEntityManager implements EntityManager {
collMgr.load( edge.getSourceNode() ).toBlockingObservable().last();
if (collEntity == null) {
- if (logger.isDebugEnabled()) {
- logger.error("FAILED to load entity {}:{} "
- + "from scope\n app {}\n owner {}\n name {}",
- new Object[]{
- edge.getSourceNode().getType(),
- edge.getSourceNode().getUuid(),
- collScope.getApplication(),
- collScope.getOwner(),
- collScope.getName()
- });
- }
+ logger.warn("(Empty collection?) Failed to load collection entity "
+ + "{}:{} from scope\n app {}\n owner {}\n name {}",
+ new Object[]{
+ edge.getSourceNode().getType(),
+ edge.getSourceNode().getUuid(),
+ collScope.getApplication(),
+ collScope.getOwner(),
+ collScope.getName()
+ });
return;
}
@@ -2872,17 +2870,15 @@ public class CpEntityManager implements EntityManager {
memberMgr.load( edge.getTargetNode()).toBlockingObservable().last();
if (memberEntity == null) {
- if (logger.isDebugEnabled()) {
- logger.error("FAILED to load entity {}:{} "
- + "from scope\n app {}\n owner {}\n name {}",
- new Object[]{
- edge.getTargetNode().getType(),
- edge.getTargetNode().getUuid(),
- memberScope.getApplication(),
- memberScope.getOwner(),
- memberScope.getName()
- });
- }
+ logger.warn("(Empty collection?) Failed to load member entity "
+ + "{}:{} from scope\n app {}\n owner {}\n name {}",
+ new Object[]{
+ edge.getTargetNode().getType(),
+ edge.getTargetNode().getUuid(),
+ memberScope.getApplication(),
+ memberScope.getOwner(),
+ memberScope.getName()
+ });
return;
}
@@ -2917,17 +2913,15 @@ public class CpEntityManager implements EntityManager {
sourceEcm.load( edge.getSourceNode() ).toBlockingObservable().last();
if (sourceEntity == null) {
- if (logger.isDebugEnabled()) {
- logger.error("FAILED to load entity {}:{} "
- + "from scope\n app {}\n owner {}\n name {}",
- new Object[]{
- edge.getSourceNode().getType(),
- edge.getSourceNode().getUuid(),
- sourceScope.getApplication(),
- sourceScope.getOwner(),
- sourceScope.getName()
- });
- }
+ logger.warn("(Empty connection?) Failed to load source entity "
+ + "{}:{} from scope\n app {}\n owner {}\n name {}",
+ new Object[]{
+ edge.getSourceNode().getType(),
+ edge.getSourceNode().getUuid(),
+ sourceScope.getApplication(),
+ sourceScope.getOwner(),
+ sourceScope.getName()
+ });
return;
}
@@ -2942,17 +2936,15 @@ public class CpEntityManager implements EntityManager {
targetEcm.load( edge.getTargetNode() ).toBlockingObservable().last();
if (targetEntity == null) {
- if (logger.isDebugEnabled()) {
- logger.error("FAILED to load entity {}:{} "
- + "from scope\n app {}\n owner {}\n name {}",
- new Object[]{
- edge.getTargetNode().getType(),
- edge.getTargetNode().getUuid(),
- targetScope.getApplication(),
- targetScope.getOwner(),
- targetScope.getName()
- });
- }
+ logger.warn("(Empty connection?) Failed to load target entity "
+ + "{}:{} from scope\n app {}\n owner {}\n name {}",
+ new Object[]{
+ edge.getTargetNode().getType(),
+ edge.getTargetNode().getUuid(),
+ targetScope.getApplication(),
+ targetScope.getOwner(),
+ targetScope.getName()
+ });
return;
}
[6/6] git commit: Merge branch 'two-dot-o' of
https://git-wip-us.apache.org/repos/asf/incubator-usergrid into map
Posted by sf...@apache.org.
Merge branch 'two-dot-o' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into map
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/587cb5a7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/587cb5a7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/587cb5a7
Branch: refs/heads/map
Commit: 587cb5a73d636988008b9f4600404ff67b451c4c
Parents: 9a8bed5 cff6e1c
Author: Shawn Feldman <sf...@apache.org>
Authored: Thu Oct 2 11:45:51 2014 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Thu Oct 2 11:45:51 2014 -0600
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 106 ++++++++++++-------
.../corepersistence/CpEntityManagerFactory.java | 2 -
2 files changed, 70 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
[4/6] git commit: Merge branch 'two-dot-o' of
https://git-wip-us.apache.org/repos/asf/incubator-usergrid into two-dot-o
Posted by sf...@apache.org.
Merge branch 'two-dot-o' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into two-dot-o
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/878d634c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/878d634c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/878d634c
Branch: refs/heads/map
Commit: 878d634ca26c152f7112fd312d27321afe05a891
Parents: 1073f11 b644cc0
Author: Dave Johnson <dm...@apigee.com>
Authored: Thu Oct 2 12:18:14 2014 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Thu Oct 2 12:18:14 2014 -0400
----------------------------------------------------------------------
stack/pom.xml | 2 +-
.../java/org/apache/usergrid/rest/PartialUpdateTest.java | 9 ++++++---
2 files changed, 7 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
[3/6] git commit: Additional debugging.
Posted by sf...@apache.org.
Additional debugging.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/1073f115
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/1073f115
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/1073f115
Branch: refs/heads/map
Commit: 1073f11588a55f0718325ec66f23800546aeda8f
Parents: a6665f1
Author: Dave Johnson <dm...@apigee.com>
Authored: Thu Oct 2 12:17:52 2014 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Thu Oct 2 12:17:52 2014 -0400
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 88 +++++++++++++-------
.../corepersistence/CpEntityManagerFactory.java | 2 -
2 files changed, 59 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1073f115/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 8d3940d..f618a80 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -2846,20 +2846,20 @@ public class CpEntityManager implements EntityManager {
org.apache.usergrid.persistence.model.entity.Entity collEntity =
collMgr.load( edge.getSourceNode() ).toBlockingObservable().last();
-// if (collEntity == null) {
-// if (logger.isDebugEnabled()) {
-// logger.error("FAILED to load entity {}:{} "
-// + "from scope\n app {}\n owner {}\n name {}",
-// new Object[]{
-// edge.getSourceNode().getType(),
-// edge.getSourceNode().getUuid(),
-// collScope.getApplication(),
-// collScope.getOwner(),
-// collScope.getName()
-// });
-// }
-// return;
-// }
+ if (collEntity == null) {
+ if (logger.isDebugEnabled()) {
+ logger.error("FAILED to load entity {}:{} "
+ + "from scope\n app {}\n owner {}\n name {}",
+ new Object[]{
+ edge.getSourceNode().getType(),
+ edge.getSourceNode().getUuid(),
+ collScope.getApplication(),
+ collScope.getOwner(),
+ collScope.getName()
+ });
+ }
+ return;
+ }
CollectionScope memberScope = new CollectionScopeImpl(
applicationScope.getApplication(),
@@ -2871,20 +2871,20 @@ public class CpEntityManager implements EntityManager {
org.apache.usergrid.persistence.model.entity.Entity memberEntity =
memberMgr.load( edge.getTargetNode()).toBlockingObservable().last();
-// if (memberEntity == null) {
-// if (logger.isDebugEnabled()) {
-// logger.error("FAILED to load entity {}:{} "
-// + "from scope\n app {}\n owner {}\n name {}",
-// new Object[]{
-// edge.getTargetNode().getType(),
-// edge.getTargetNode().getUuid(),
-// memberScope.getApplication(),
-// memberScope.getOwner(),
-// memberScope.getName()
-// });
-// }
-// return;
-// }
+ if (memberEntity == null) {
+ if (logger.isDebugEnabled()) {
+ logger.error("FAILED to load entity {}:{} "
+ + "from scope\n app {}\n owner {}\n name {}",
+ new Object[]{
+ edge.getTargetNode().getType(),
+ edge.getTargetNode().getUuid(),
+ memberScope.getApplication(),
+ memberScope.getOwner(),
+ memberScope.getName()
+ });
+ }
+ return;
+ }
indexEntityIntoCollections( collEntity, memberEntity, collName, true );
@@ -2914,7 +2914,22 @@ public class CpEntityManager implements EntityManager {
managerCache.getEntityCollectionManager(sourceScope);
org.apache.usergrid.persistence.model.entity.Entity sourceEntity =
- sourceEcm.load( fromEntityId ).toBlockingObservable().last();
+ sourceEcm.load( edge.getSourceNode() ).toBlockingObservable().last();
+
+ if (sourceEntity == null) {
+ if (logger.isDebugEnabled()) {
+ logger.error("FAILED to load entity {}:{} "
+ + "from scope\n app {}\n owner {}\n name {}",
+ new Object[]{
+ edge.getSourceNode().getType(),
+ edge.getSourceNode().getUuid(),
+ sourceScope.getApplication(),
+ sourceScope.getOwner(),
+ sourceScope.getName()
+ });
+ }
+ return;
+ }
CollectionScope targetScope = new CollectionScopeImpl(
applicationScope.getApplication(),
@@ -2926,6 +2941,21 @@ public class CpEntityManager implements EntityManager {
org.apache.usergrid.persistence.model.entity.Entity targetEntity =
targetEcm.load( edge.getTargetNode() ).toBlockingObservable().last();
+ if (targetEntity == null) {
+ if (logger.isDebugEnabled()) {
+ logger.error("FAILED to load entity {}:{} "
+ + "from scope\n app {}\n owner {}\n name {}",
+ new Object[]{
+ edge.getTargetNode().getType(),
+ edge.getTargetNode().getUuid(),
+ targetScope.getApplication(),
+ targetScope.getOwner(),
+ targetScope.getName()
+ });
+ }
+ return;
+ }
+
indexEntityIntoConnection(
sourceEntity, targetEntity, targetEntityType, connType );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1073f115/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index 845d798..26f4234 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -52,8 +52,6 @@ import org.apache.usergrid.persistence.graph.GraphManagerFactory;
import org.apache.usergrid.persistence.graph.SearchByEdgeType;
import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdgeType;
import org.apache.usergrid.persistence.index.EntityIndexFactory;
-import org.apache.usergrid.persistence.index.IndexScope;
-import org.apache.usergrid.persistence.index.impl.IndexScopeImpl;
import org.apache.usergrid.persistence.index.query.Query;
import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.entity.SimpleId;