You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sn...@apache.org on 2014/08/22 20:48:28 UTC

[1/3] git commit: Make sure Tomcat gets the ulimit.

Repository: incubator-usergrid
Updated Branches:
  refs/heads/two-dot-o ac0ca5de0 -> 839ac295e


Make sure Tomcat gets the ulimit.


Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/d09bb712
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/d09bb712
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/d09bb712

Branch: refs/heads/two-dot-o
Commit: d09bb71217e2f63defe848187055c16a96f1c663
Parents: ac0ca5d
Author: Dave Johnson <dm...@apigee.com>
Authored: Thu Aug 21 16:16:11 2014 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Thu Aug 21 16:16:11 2014 -0400

----------------------------------------------------------------------
 stack/awscluster/src/main/dist/init_instance/init_rest_server.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d09bb712/stack/awscluster/src/main/dist/init_instance/init_rest_server.sh
----------------------------------------------------------------------
diff --git a/stack/awscluster/src/main/dist/init_instance/init_rest_server.sh b/stack/awscluster/src/main/dist/init_instance/init_rest_server.sh
index be95a51..7a8954e 100644
--- a/stack/awscluster/src/main/dist/init_instance/init_rest_server.sh
+++ b/stack/awscluster/src/main/dist/init_instance/init_rest_server.sh
@@ -80,7 +80,6 @@ case `(curl http://169.254.169.254/latest/meta-data/instance-type)` in
 'c3.4xlarge' )
     export TOMCAT_RAM=24G
     export TOMCAT_THREADS=4000
-    export NOFILE=200000
 esac
 
 export TOMCAT_CONNECTIONS=10000
@@ -88,6 +87,7 @@ sudo sed -i.bak "s/Xmx128m/Xmx${TOMCAT_RAM} -Xms${TOMCAT_RAM}/g" /etc/default/to
 sudo sed -i.bak "s/<Connector/<Connector maxThreads=\"${TOMCAT_THREADS}\" acceptCount=\"${TOMCAT_THREADS}\" maxConnections=\"${TOMCAT_CONNECTIONS}\"/g" /var/lib/tomcat7/conf/server.xml
 
 # set file limits
+sudo sed -i.bak "s/# \/etc\/init\.d\/tomcat7 -- startup script for the Tomcat 6 servlet engine/ulimit ${NOFILE}/" /etc/init.d/tomcat7
 sudo sed -i.bak "/@student/a *\t\thard\tnofile\t\t${NOFILE}\n*\t\tsoft\tnofile\t\t${NOFILE}" /etc/security/limits.conf
 echo "$NOFILE" | sudo tee > /proc/sys/fs/nr_open
 echo "$NOFILE" | sudo tee > /proc/sys/fs/file-max


[3/3] git commit: Add support for c3.2xlarge to AWS Cluster template.

Posted by sn...@apache.org.
Add support for c3.2xlarge to AWS Cluster template.


Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/839ac295
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/839ac295
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/839ac295

Branch: refs/heads/two-dot-o
Commit: 839ac295e20d8f214fbfa9536d3afa7e9f2b1e4b
Parents: bc3ad2a
Author: Dave Johnson <dm...@apigee.com>
Authored: Fri Aug 22 14:44:50 2014 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Fri Aug 22 14:44:50 2014 -0400

----------------------------------------------------------------------
 .../src/main/dist/init_instance/init_rest_server.sh       |  4 ++++
 stack/awscluster/ugcluster-cf.json                        | 10 ++++++++--
 2 files changed, 12 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/839ac295/stack/awscluster/src/main/dist/init_instance/init_rest_server.sh
----------------------------------------------------------------------
diff --git a/stack/awscluster/src/main/dist/init_instance/init_rest_server.sh b/stack/awscluster/src/main/dist/init_instance/init_rest_server.sh
index 7a8954e..d069d0d 100644
--- a/stack/awscluster/src/main/dist/init_instance/init_rest_server.sh
+++ b/stack/awscluster/src/main/dist/init_instance/init_rest_server.sh
@@ -77,6 +77,10 @@ case `(curl http://169.254.169.254/latest/meta-data/instance-type)` in
     export TOMCAT_RAM=6G
     export TOMCAT_THREADS=1600
 ;;
+'c3.2xlarge' )
+    export TOMCAT_RAM=12G
+    export TOMCAT_THREADS=2000
+;;
 'c3.4xlarge' )
     export TOMCAT_RAM=24G
     export TOMCAT_THREADS=4000

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/839ac295/stack/awscluster/ugcluster-cf.json
----------------------------------------------------------------------
diff --git a/stack/awscluster/ugcluster-cf.json b/stack/awscluster/ugcluster-cf.json
index 9c4885f..1de4e54 100644
--- a/stack/awscluster/ugcluster-cf.json
+++ b/stack/awscluster/ugcluster-cf.json
@@ -40,6 +40,7 @@
                 "m1.xlarge",
                 "m3.xlarge",
                 "m3.large",
+                "c3.2xlarge",
                 "c3.4xlarge"
             ],
             "ConstraintDescription": "must be valid instance type."
@@ -63,7 +64,7 @@
         "CassInstanceType": {
             "Description": "Instance type for Cass servers",
             "Type": "String",
-            "Default": "c3.4xlarge",
+            "Default": "c3.2xlarge",
             "AllowedValues": [
                 "m1.small",
                 "m1.medium",
@@ -71,6 +72,7 @@
                 "m1.xlarge",
                 "m3.xlarge",
                 "m3.2xlarge",
+                "c3.2xlarge",
                 "c3.4xlarge"
             ],
             "ConstraintDescription": "must be valid instance type."
@@ -151,6 +153,9 @@
             "c3.2xlarge": {
                 "Arch": "64"
             },
+            "c3.2xlarge": {
+                "Arch": "64"
+            },
             "c3.4xlarge": {
                 "Arch": "64"
             }
@@ -857,7 +862,8 @@
                     "UnhealthyThreshold": "8",
                     "Interval": "30",
                     "Timeout": "5"
-                }
+                },
+                "CrossZone": "true"
             }
         },
         "NotificationTopic": {


[2/3] git commit: Fix to "reindex on update" logic that allows the ServiceInvocationIT tests to pass.

Posted by sn...@apache.org.
Fix  to "reindex on update" logic that allows the ServiceInvocationIT tests to pass.


Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/bc3ad2a3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/bc3ad2a3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/bc3ad2a3

Branch: refs/heads/two-dot-o
Commit: bc3ad2a354cdb285ce1ec57d1df34a66c454eedf
Parents: d09bb71
Author: Dave Johnson <dm...@apigee.com>
Authored: Thu Aug 21 16:17:02 2014 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Thu Aug 21 16:17:02 2014 -0400

----------------------------------------------------------------------
 .../corepersistence/CpEntityManager.java        |  63 ++---------
 .../corepersistence/CpRelationManager.java      | 108 ++++++++++++++++---
 .../usergrid/services/ServiceInvocationIT.java  |  49 ++++-----
 3 files changed, 121 insertions(+), 99 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bc3ad2a3/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index b216294..2ce3d56 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -520,61 +520,9 @@ public class CpEntityManager implements EntityManager {
             }
         }
         
-        // update item in collection index
-        IndexScope indexScope = new IndexScopeImpl(
-            appScope.getApplication(), 
-            appScope.getApplication(), 
-            getCollectionScopeNameFromEntityType( entity.getType() ));
-        EntityIndex ei = managerCache.getEntityIndex( indexScope );
-        ei.index( cpEntity );
-
-        // update all items index
-        IndexScope allTypesIndexScope = new IndexScopeImpl( 
-            appScope.getApplication(), 
-            appScope.getApplication(), 
-            ALL_TYPES);
-        EntityIndex aei = managerCache.getEntityIndex( allTypesIndexScope );
-        aei.index( cpEntity );
-
-        // next, update entity in every collection and connection scope in which it is indexed 
-        updateEntityIndexes( entity, cpEntity );
-    }
-
-
-    private void updateEntityIndexes( 
-            Entity entity, org.apache.usergrid.persistence.model.entity.Entity cpEntity )
-            throws Exception {
-
-        RelationManager rm = getRelationManager( entity );
-        Map<String, Map<UUID, Set<String>>> owners = rm.getOwners();
-
-        logger.debug( "Updating indexes of all {} collections owning the entity", 
-                owners.keySet().size() );
-
-        for ( String ownerType : owners.keySet() ) {
-            Map<UUID, Set<String>> collectionsByUuid = owners.get( ownerType );
-
-            for ( UUID uuid : collectionsByUuid.keySet() ) {
-                Set<String> collections = collectionsByUuid.get( uuid );
-                for ( String coll : collections ) {
-
-                    if ( coll == null || coll.trim().isEmpty() ) {
-                        logger.warn( "Ignoring empty collection name for owner {}:{}", 
-                                uuid, ownerType );
-                        break;
-                    }
-
-                    IndexScope indexScope = new IndexScopeImpl( 
-                        appScope.getApplication(), 
-                        new SimpleId( uuid, ownerType ),
-                        getCollectionScopeNameFromCollectionName( coll ) );
-
-                    EntityIndex ei = managerCache.getEntityIndex( indexScope );
-
-                    ei.index( cpEntity );
-                }
-            }
-        }
+        // update in all containing collections and connection indexes
+        CpRelationManager rm = (CpRelationManager)getRelationManager( entity );
+        rm.updateContainingCollectionAndCollectionIndexes( entity, cpEntity );
     }
 
 
@@ -1056,8 +1004,9 @@ public class CpEntityManager implements EntityManager {
 
         ei.index( cpEntity );
 
-        // update entity in every collection and connection scope in which it is indexed
-        updateEntityIndexes( get( entityRef ), cpEntity );
+        // update in all containing collections and connection indexes
+        CpRelationManager rm = (CpRelationManager)getRelationManager( entityRef );
+        rm.updateContainingCollectionAndCollectionIndexes( get( entityRef ), cpEntity );
     }
 
 

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bc3ad2a3/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index ad6041d..d467682 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -271,16 +271,24 @@ public class CpRelationManager implements RelationManager {
     }
 
 
-    static boolean isConnectionEdgeType( String type )  {
+    static boolean isCollectionEdgeType( String type )  {
         return type.endsWith( EDGE_COLL_SUFFIX );
     }
-
+    
+    static boolean isConnectionEdgeType( String type )  {
+        return type.endsWith( EDGE_CONN_SUFFIX );
+    }
     
     public String getConnectionName( String edgeType ) {
         String[] parts = edgeType.split("\\|");
         return parts[0];
     }
 
+    public String getCollectionName( String edgeType ) {
+        String[] parts = edgeType.split("\\|");
+        return parts[0];
+    }
+
 
     @Override
     public Set<String> getCollectionIndexes( String collectionName ) throws Exception {
@@ -376,11 +384,13 @@ public class CpRelationManager implements RelationManager {
                 EntityRef eref = new SimpleEntityRef( 
                     edge.getSourceNode().getType(), edge.getSourceNode().getUuid() );
 
-                String connectionName = null;
+                String name = null;
                 if ( isConnectionEdgeType( edge.getType() )) {
-                    connectionName = getConnectionName( edge.getType() );
+                    name = getConnectionName( edge.getType() );
+                } else {
+                    name = getCollectionName( edge.getType() );
                 }
-                addMapSet( results, eref, connectionName );
+                addMapSet( results, eref, name );
             }
 
             if ( limit > 0 && results.keySet().size() >= limit ) {
@@ -388,16 +398,84 @@ public class CpRelationManager implements RelationManager {
             }
         }
 
-        // should not need to do this
-//        if ( connType == null ) {
-//
-//            EntityRef applicationRef = new SimpleEntityRef( TYPE_APPLICATION, applicationId );
-//            if ( !results.containsKey( applicationRef ) ) {
-//
-//                addMapSet( results, applicationRef, 
-//                    CpEntityManager.getCollectionScopeNameFromEntityType( headEntity.getType() ) );
-//            }
-//        }
+        return results;
+    }
+
+
+    public List<String> updateContainingCollectionAndCollectionIndexes( 
+        Entity entity, org.apache.usergrid.persistence.model.entity.Entity cpEntity ) {
+
+        List<String> results = new ArrayList<String>();
+
+        GraphManager gm = managerCache.getGraphManager(applicationScope);
+
+        Iterator<String> edgeTypesToTarget = gm.getEdgeTypesToTarget( new SimpleSearchEdgeType( 
+            cpHeadEntity.getId(), null, null) ).toBlockingObservable().getIterator();
+
+        logger.debug("updateContainingCollectionsAndCollections(): "
+                + "Searched for edges to target {}:{}\n   in scope {}\n   found: {}", 
+            new Object[] {
+                cpHeadEntity.getId().getType(), 
+                cpHeadEntity.getId().getUuid(), 
+                applicationScope.getApplication(),
+                edgeTypesToTarget.hasNext()
+        });
+
+        // loop through all types of edge to target
+        int count = 0;
+        while ( edgeTypesToTarget.hasNext() ) {
+
+            // get all edges of the type
+            String etype = edgeTypesToTarget.next();
+
+            Observable<Edge> edges = gm.loadEdgesToTarget( new SimpleSearchByEdgeType(
+                cpHeadEntity.getId(), etype, Long.MAX_VALUE, null ));
+
+            // loop through edges of that type
+            Iterator<Edge> iter = edges.toBlockingObservable().getIterator();
+            while ( iter.hasNext() ) {
+
+                Edge edge = iter.next();
+
+                EntityRef sourceEntity = new SimpleEntityRef( 
+                    edge.getSourceNode().getType(), edge.getSourceNode().getUuid() );
+
+                // reindex the entity in the source entity's collection or connection index
+
+                IndexScope indexScope;
+                if ( isCollectionEdgeType( edge.getType() )) {
+
+                    String collName = getCollectionName( edge.getType() ); 
+                    indexScope = new IndexScopeImpl(
+                        applicationScope.getApplication(),
+                        new SimpleId(sourceEntity.getUuid(), sourceEntity.getType()),
+                        CpEntityManager.getCollectionScopeNameFromCollectionName( collName ));
+
+                } else {
+
+                    String connName = getCollectionName( edge.getType() ); 
+                    indexScope = new IndexScopeImpl(
+                        applicationScope.getApplication(),
+                        new SimpleId(sourceEntity.getUuid(), sourceEntity.getType()),
+                        CpEntityManager.getConnectionScopeName( cpHeadEntity.getId().getType(), connName ));
+                } 
+           
+                EntityIndex ei = managerCache.getEntityIndex(indexScope);
+                ei.index(cpEntity);
+
+                // reindex the entity in the source entity's all-types index
+                
+                indexScope = new IndexScopeImpl(
+                    applicationScope.getApplication(),
+                    new SimpleId(sourceEntity.getUuid(), sourceEntity.getType()),
+                    ALL_TYPES);
+                ei = managerCache.getEntityIndex(indexScope);
+                ei.index(cpEntity);
+
+                count++;
+            }
+        }
+        logger.debug("updateContainingCollectionsAndCollections() updated {} indexes", count);
         return results;
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bc3ad2a3/stack/services/src/test/java/org/apache/usergrid/services/ServiceInvocationIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/services/ServiceInvocationIT.java b/stack/services/src/test/java/org/apache/usergrid/services/ServiceInvocationIT.java
index 5d1bf25..bde23a5 100644
--- a/stack/services/src/test/java/org/apache/usergrid/services/ServiceInvocationIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/services/ServiceInvocationIT.java
@@ -22,15 +22,13 @@ import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.UUID;
-
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 import org.apache.usergrid.persistence.Entity;
 import org.apache.usergrid.persistence.index.query.Query;
 import org.apache.usergrid.persistence.model.util.UUIDGenerator;
-
 import static org.junit.Assert.assertNotNull;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 
 public class ServiceInvocationIT extends AbstractServiceIT {
@@ -102,28 +100,25 @@ public class ServiceInvocationIT extends AbstractServiceIT {
 
         app.testRequest( ServiceAction.PUT, 1, "users", "edanuff", "likes", cat.getUuid() );
 
-        // TODO: uncomment this code and fix whatever problem is causing it to fail
-        // see also: https://issues.apache.org/jira/browse/USERGRID-214
-
-//        app.put( "eats", "petfood" );
-//
-//        app.testRequest( ServiceAction.PUT, 1, "users", "edanuff", "likes", "cats", "dylan" );
-//
-//        app.put( "Todays special", "Coffee" );
-//
-//        app.testRequest( ServiceAction.PUT, 1, "users", "edanuff", "likes", "restaurants",
-//                Query.fromQL( "select * where name='Brickhouse'" ) );
-//
-//        app.testRequest( ServiceAction.DELETE, 1, null, "users", user.getUuid(), "connections", "likes",
-//                restaurant.getUuid() );
-//
-//        app.testRequest( ServiceAction.GET, 2, null, "users", "edanuff", "connections" );
-//
-//        app.testRequest( ServiceAction.GET, 1, null, "users", "edanuff", "likes", "restaurants" );
-//
-//        UUID uuid = UUIDGenerator.newTimeUUID();
-//        app.put( "visits", 5 );
-//        app.testRequest( ServiceAction.PUT, 1, "devices", uuid );
+        app.put( "eats", "petfood" );
+
+        app.testRequest( ServiceAction.PUT, 1, "users", "edanuff", "likes", "cats", "dylan" );
+
+        app.put( "Todays special", "Coffee" );
+
+        app.testRequest( ServiceAction.PUT, 1, "users", "edanuff", "likes", "restaurants",
+                Query.fromQL( "select * where name='Brickhouse'" ) );
+
+        app.testRequest( ServiceAction.DELETE, 1, null, "users", user.getUuid(), "connections", "likes",
+                restaurant.getUuid() );
+
+        app.testRequest( ServiceAction.GET, 2, null, "users", "edanuff", "connections" );
+
+        app.testRequest( ServiceAction.GET, 1, null, "users", "edanuff", "likes", "restaurants" );
+
+        UUID uuid = UUIDGenerator.newTimeUUID();
+        app.put( "visits", 5 );
+        app.testRequest( ServiceAction.PUT, 1, "devices", uuid );
     }