You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sn...@apache.org on 2014/08/15 14:56:37 UTC
git commit: Fixes to Tomcat and Cassandra thread logic,
also adding wait to NodeRegistry to work around apparent AWS SimpleDB
throttling.
Repository: incubator-usergrid
Updated Branches:
refs/heads/two-dot-o 5878b4060 -> 44d1c8b53
Fixes to Tomcat and Cassandra thread logic, also adding wait to NodeRegistry to work around apparent AWS SimpleDB throttling.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/44d1c8b5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/44d1c8b5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/44d1c8b5
Branch: refs/heads/two-dot-o
Commit: 44d1c8b533a8fddd1e55c7c57652704aa9553c35
Parents: 5878b40
Author: Dave Johnson <dm...@apigee.com>
Authored: Fri Aug 15 08:52:12 2014 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Fri Aug 15 08:52:12 2014 -0400
----------------------------------------------------------------------
.../main/dist/init_instance/init_rest_server.sh | 43 +++++++++++---------
.../src/main/groovy/NodeRegistry.groovy | 3 +-
.../src/main/groovy/configure_cassandra.groovy | 10 ++++-
.../src/main/groovy/configure_usergrid.groovy | 11 +++--
stack/awscluster/ugcluster-cf.json | 2 +-
5 files changed, 43 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/44d1c8b5/stack/awscluster/src/main/dist/init_instance/init_rest_server.sh
----------------------------------------------------------------------
diff --git a/stack/awscluster/src/main/dist/init_instance/init_rest_server.sh b/stack/awscluster/src/main/dist/init_instance/init_rest_server.sh
index 97d7082..4dafe0a 100644
--- a/stack/awscluster/src/main/dist/init_instance/init_rest_server.sh
+++ b/stack/awscluster/src/main/dist/init_instance/init_rest_server.sh
@@ -49,18 +49,6 @@ chmod +x /usr/share/usergrid/update.sh
cd /usr/share/usergrid/init_instance
./install_oraclejdk.sh
-# Wait for enough Cassandra nodes then deploy and restart Tomcat
-cd /usr/share/usergrid/scripts
-groovy wait_for_instances.groovy cassandra ${CASSANDRA_NUM_SERVERS}
-groovy wait_for_instances.groovy graphite ${GRAPHITE_NUM_SERVERS}
-
-mkdir -p /usr/share/tomcat7/lib
-groovy configure_usergrid.groovy > /usr/share/tomcat7/lib/usergrid-custom.properties
-
-rm -rf /var/lib/tomcat7/webapps/*
-cp -r /usr/share/usergrid/webapps/* /var/lib/tomcat7/webapps
-groovy configure_portal_new.groovy >> /var/lib/tomcat7/webapps/portal/config.js
-
cd /usr/share/usergrid/init_instance
./install_yourkit.sh
@@ -70,10 +58,11 @@ ln -s /var/log varlog/tomcat7 tomcat7logs
ln -s /usr/share/tomcat7 tomcat7
ln -s /usr/share/usergrid usergrid
+# set Tomcat memory and threads based on instance type
case `(curl http://169.254.169.254/latest/meta-data/instance-type)` in
'm1.small' )
export TOMCAT_RAM=1250M
- export TOMCAT_THREADS=250
+ export TOMCAT_THREADS=300
;;
'm1.medium' )
export TOMCAT_RAM=3G
@@ -89,7 +78,7 @@ case `(curl http://169.254.169.254/latest/meta-data/instance-type)` in
;;
'm3.xlarge' )
export TOMCAT_RAM=12G
- export TOMCAT_THREADS=3250
+ export TOMCAT_THREADS=3300
;;
'm3.large' )
export TOMCAT_RAM=6G
@@ -97,15 +86,31 @@ case `(curl http://169.254.169.254/latest/meta-data/instance-type)` in
;;
'c3.4xlarge' )
export TOMCAT_RAM=24G
- export TOMCAT_THREADS=15000
- export NOFILE=400000
+ export TOMCAT_THREADS=4000
+ export NOFILE=700000
esac
+# set file limits high
+export TOMCAT_CONNECTIONS=10000
sudo sed -i.bak "s/Xmx128m/Xmx${TOMCAT_RAM} -Xms${TOMCAT_RAM}/g" /etc/default/tomcat7
-sudo sed -i.bak "s/<Connector/<Connector maxThreads=\"${TOMCAT_THREADS}\" acceptCount=\"${TOMCAT_THREADS}\" maxConnections=\"${TOMCAT_THREADS}\"/g" /var/lib/tomcat7/conf/server.xml
+sudo sed -i.bak "s/<Connector/<Connector maxThreads=\"${TOMCAT_THREADS}\" acceptCount=\"${TOMCAT_THREADS}\" maxConnections=\"${TOMCAT_CONNECTIONS}\"/g" /var/lib/tomcat7/conf/server.xml
sudo sed -i.bak "/@student/a *\t\thard\tnofile\t\t${NOFILE}\n*\t\tsoft\tnofile\t\t${NOFILE}" /etc/security/limits.conf
-echo 600000 | sudo tee /proc/sys/fs/nr_open
-echo 600000 | sudo tee /proc/sys/fs/file-max
+echo 700000 | sudo tee /proc/sys/fs/nr_open
+echo 700000 | sudo tee /proc/sys/fs/file-max
+export NOFILE=700000
+
+# Wait for enough Cassandra nodes then deploy and restart Tomcat
+cd /usr/share/usergrid/scripts
+groovy wait_for_instances.groovy cassandra ${CASSANDRA_NUM_SERVERS}
+groovy wait_for_instances.groovy graphite ${GRAPHITE_NUM_SERVERS}
+
+rm -rf /var/lib/tomcat7/webapps/*
+cp -r /usr/share/usergrid/webapps/* /var/lib/tomcat7/webapps
+groovy configure_portal_new.groovy >> /var/lib/tomcat7/webapps/portal/config.js
+
+# configure usergrid
+mkdir -p /usr/share/tomcat7/lib
+groovy configure_usergrid.groovy > /usr/share/tomcat7/lib/usergrid-custom.properties
# tag last so we can see in the console that the script ran to completion
cd /usr/share/usergrid/scripts
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/44d1c8b5/stack/awscluster/src/main/groovy/NodeRegistry.groovy
----------------------------------------------------------------------
diff --git a/stack/awscluster/src/main/groovy/NodeRegistry.groovy b/stack/awscluster/src/main/groovy/NodeRegistry.groovy
index 38c44be..f6394c2 100644
--- a/stack/awscluster/src/main/groovy/NodeRegistry.groovy
+++ b/stack/awscluster/src/main/groovy/NodeRegistry.groovy
@@ -48,7 +48,8 @@ class NodeRegistry {
* @param defNodeType
*/
def searchNode(def nodeType) {
- def selectResult = sdbClient.select(new SelectRequest((String) "select * from `${domain}` where itemName() is not null and nodetype = '${nodeType}' order by itemName()"))
+ def selectResult = sdbClient.select(new SelectRequest((String) \
+ "select * from `${domain}` where itemName() is not null and nodetype = '${nodeType}' order by itemName()"))
def result = []
for (item in selectResult.getItems()) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/44d1c8b5/stack/awscluster/src/main/groovy/configure_cassandra.groovy
----------------------------------------------------------------------
diff --git a/stack/awscluster/src/main/groovy/configure_cassandra.groovy b/stack/awscluster/src/main/groovy/configure_cassandra.groovy
index 9e5ded2..161b1c8 100644
--- a/stack/awscluster/src/main/groovy/configure_cassandra.groovy
+++ b/stack/awscluster/src/main/groovy/configure_cassandra.groovy
@@ -34,7 +34,15 @@ int cassNumServers = ((String)System.getenv().get("CASSANDRA_NUM_SERVERS")).toIn
// build seed list by listing all Cassandra nodes found in SimpleDB domain with our stackName
-NodeRegistry registry = new NodeRegistry();
+NodeRegistry registry = null
+while ( registry == null ) {
+ try {
+ registry = new NodeRegistry()
+ } catch ( Exception e ) {
+ Thread.sleep(2000);
+ continue;
+ }
+}
def selectResult = registry.searchNode('cassandra')
def seeds = ""
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/44d1c8b5/stack/awscluster/src/main/groovy/configure_usergrid.groovy
----------------------------------------------------------------------
diff --git a/stack/awscluster/src/main/groovy/configure_usergrid.groovy b/stack/awscluster/src/main/groovy/configure_usergrid.groovy
index d793a81..cdbbc77 100644
--- a/stack/awscluster/src/main/groovy/configure_usergrid.groovy
+++ b/stack/awscluster/src/main/groovy/configure_usergrid.groovy
@@ -40,6 +40,9 @@ def clusterName = System.getenv().get("CASSANDRA_CLUSTER_NAME")
def superUserEmail = System.getenv().get("SUPER_USER_EMAIL")
def testAdminUserEmail = System.getenv().get("TEST_ADMIN_USER_EMAIL")
+def cassThreads = System.getenv().get("TOMCAT_THREADS")
+def hystrixThreads = Integer.parseInt(cassThreads) / 100
+
NodeRegistry registry = new NodeRegistry();
@@ -80,10 +83,10 @@ cassandra.cluster=${clusterName}
cassandra.keyspace.strategy=org.apache.cassandra.locator.SimpleStrategy
cassandra.keyspace.replication=${replFactor}
-cassandra.timeout=2000
-cassandra.connections=800
-hystrix.threadpool.graph_user.coreSize=60
-hystrix.threadpool.graph_async.coreSize=60
+cassandra.timeout=5000
+cassandra.connections=${cassThreads}
+hystrix.threadpool.graph_user.coreSize=${hystrixThreads}
+hystrix.threadpool.graph_async.coreSize=${hystrixThreads}
elasticsearch.cluster_name=${clusterName}
elasticsearch.index_prefix=usergrid
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/44d1c8b5/stack/awscluster/ugcluster-cf.json
----------------------------------------------------------------------
diff --git a/stack/awscluster/ugcluster-cf.json b/stack/awscluster/ugcluster-cf.json
index ef655e1..9c4885f 100644
--- a/stack/awscluster/ugcluster-cf.json
+++ b/stack/awscluster/ugcluster-cf.json
@@ -854,7 +854,7 @@
"HealthCheck": {
"Target": "HTTP:8080/status",
"HealthyThreshold": "2",
- "UnhealthyThreshold": "4",
+ "UnhealthyThreshold": "8",
"Interval": "30",
"Timeout": "5"
}