You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by to...@apache.org on 2015/02/06 02:48:40 UTC
[1/2] incubator-usergrid git commit: Updated setup to allow
configuring client consistency level,
as well as replication factor on keyspaces
Repository: incubator-usergrid
Updated Branches:
refs/heads/two-dot-o-import 7d27c230f -> 0fb13ee0b
Updated setup to allow configuring client consistency level, as well as replication factor on keyspaces
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/3930e85f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/3930e85f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/3930e85f
Branch: refs/heads/two-dot-o-import
Commit: 3930e85f77ec0d2975085d2a41d3ff1a16d885bb
Parents: 0bf0d81
Author: Todd Nine <tn...@apigee.com>
Authored: Thu Feb 5 18:48:26 2015 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Thu Feb 5 18:48:26 2015 -0700
----------------------------------------------------------------------
stack/awscluster/src/main/cql/update_locks.cql | 2 +-
.../awscluster/src/main/cql/update_usergrid.cql | 2 +-
.../main/cql/update_usergrid_applications.cql | 2 +-
.../dist/init_instance/install_elasticsearch.sh | 12 +++++-----
.../main/dist/init_instance/update_keyspaces.sh | 9 +++++++
.../src/main/groovy/configure_usergrid.groovy | 25 +++++++++++++-------
stack/awscluster/ugcluster-cf.json | 18 +++++++++++++-
7 files changed, 51 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3930e85f/stack/awscluster/src/main/cql/update_locks.cql
----------------------------------------------------------------------
diff --git a/stack/awscluster/src/main/cql/update_locks.cql b/stack/awscluster/src/main/cql/update_locks.cql
index 64344f6..bdb5860 100644
--- a/stack/awscluster/src/main/cql/update_locks.cql
+++ b/stack/awscluster/src/main/cql/update_locks.cql
@@ -15,7 +15,7 @@
* copyright in this work, please see the NOTICE file in the top level
* directory of this distribution.
*/
-update keyspace Locks with placement_strategy = 'NetworkTopologyStrategy' and strategy_options = {KEYSPACE_REGION : 3} and durable_writes = false;
+update keyspace Locks with placement_strategy = 'NetworkTopologyStrategy' and strategy_options = {KEYSPACE_REGION : REPLICATION_FACTOR} and durable_writes = false;
use Locks;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3930e85f/stack/awscluster/src/main/cql/update_usergrid.cql
----------------------------------------------------------------------
diff --git a/stack/awscluster/src/main/cql/update_usergrid.cql b/stack/awscluster/src/main/cql/update_usergrid.cql
index 50eb79f..53622a2 100644
--- a/stack/awscluster/src/main/cql/update_usergrid.cql
+++ b/stack/awscluster/src/main/cql/update_usergrid.cql
@@ -16,7 +16,7 @@
* directory of this distribution.
*/
-update keyspace Usergrid with placement_strategy = 'NetworkTopologyStrategy' and strategy_options = {KEYSPACE_REGION : 3} and durable_writes = true;
+update keyspace Usergrid with placement_strategy = 'NetworkTopologyStrategy' and strategy_options = {KEYSPACE_REGION : REPLICATION_FACTOR} and durable_writes = true;
use Usergrid;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3930e85f/stack/awscluster/src/main/cql/update_usergrid_applications.cql
----------------------------------------------------------------------
diff --git a/stack/awscluster/src/main/cql/update_usergrid_applications.cql b/stack/awscluster/src/main/cql/update_usergrid_applications.cql
index 253230d..e94960b 100644
--- a/stack/awscluster/src/main/cql/update_usergrid_applications.cql
+++ b/stack/awscluster/src/main/cql/update_usergrid_applications.cql
@@ -16,7 +16,7 @@
* directory of this distribution.
*/
-update keyspace Usergrid_Applications with placement_strategy = 'NetworkTopologyStrategy' and strategy_options = {KEYSPACE_REGION : 3} and durable_writes = true;
+update keyspace Usergrid_Applications with placement_strategy = 'NetworkTopologyStrategy' and strategy_options = {KEYSPACE_REGION : REPLICATION_FACTOR} and durable_writes = true;
use Usergrid_Applications;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3930e85f/stack/awscluster/src/main/dist/init_instance/install_elasticsearch.sh
----------------------------------------------------------------------
diff --git a/stack/awscluster/src/main/dist/init_instance/install_elasticsearch.sh b/stack/awscluster/src/main/dist/init_instance/install_elasticsearch.sh
index 67d9866..06af849 100644
--- a/stack/awscluster/src/main/dist/init_instance/install_elasticsearch.sh
+++ b/stack/awscluster/src/main/dist/init_instance/install_elasticsearch.sh
@@ -1,14 +1,14 @@
#!/bin/bash
-#
+#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. The ASF licenses this file to You
# under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License.
# You may obtain a copy of the License at
-#
+#
# http://www.apache.org/licenses/LICENSE-2.0
-#
+#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -23,10 +23,10 @@ pushd /etc/apt/sources.list.d
# Install and stop ElasticSearch
cat >> elasticsearch.sources.list << EOF
-deb http://packages.elasticsearch.org/elasticsearch/1.3/debian stable main
+deb http://packages.elasticsearch.org/elasticsearch/1.4/debian stable main
EOF
apt-get update
-apt-get --force-yes -y install elasticsearch
+apt-get --force-yes -y install elasticsearch=1.4.2
/etc/init.d/elasticsearch stop
mkdir -p /mnt/data/elasticsearch
@@ -49,7 +49,7 @@ groovy wait_for_instances.groovy elasticsearch ${ES_NUM_SERVERS}
# leave room for Cassandra: use about one half of RAM for heap
case `(curl http://169.254.169.254/latest/meta-data/instance-type)` in
'c3.large' )
- # total of 15g
+ # total of 15g
export ES_HEAP_SIZE=1920m
;;
'c3.xlarge' )
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3930e85f/stack/awscluster/src/main/dist/init_instance/update_keyspaces.sh
----------------------------------------------------------------------
diff --git a/stack/awscluster/src/main/dist/init_instance/update_keyspaces.sh b/stack/awscluster/src/main/dist/init_instance/update_keyspaces.sh
index c309879..95ec658 100644
--- a/stack/awscluster/src/main/dist/init_instance/update_keyspaces.sh
+++ b/stack/awscluster/src/main/dist/init_instance/update_keyspaces.sh
@@ -18,6 +18,10 @@
#
#Install keyspaces
+# Build environment for scripts
+. /etc/profile.d/aws-credentials.sh
+. /etc/profile.d/usergrid-env.sh
+
pushd /etc/apt/sources.list.d
#Run the cassandra cql to create the keyspaces. Note this only works for the
@@ -48,17 +52,22 @@ CASS_REGION=${EC2_REGION%-1}
#Update the keyspace replication and run the cql
sed -i.bak "s/KEYSPACE_REGION/${CASS_REGION}/g" /usr/share/usergrid/cql/update_locks.cql
+sed -i.bak "s/REPLICATION_FACTOR/${CASSANDRA_REPLICATION_FACTOR}/g" /usr/share/usergrid/cql/update_locks.cql
+
+
/usr/bin/cassandra-cli -h ${CASSHOST} -f /usr/share/usergrid/cql/update_locks.cql
#Update the keyspace region and run the cql
sed -i.bak "s/KEYSPACE_REGION/${CASS_REGION}/g" /usr/share/usergrid/cql/update_usergrid.cql
+sed -i.bak "s/REPLICATION_FACTOR/${CASSANDRA_REPLICATION_FACTOR}/g" /usr/share/usergrid/cql/update_usergrid.cql
/usr/bin/cassandra-cli -h ${CASSHOST} -f /usr/share/usergrid/cql/update_usergrid.cql
#Update the keyspace region and run the cql
sed -i.bak "s/KEYSPACE_REGION/${CASS_REGION}/g" /usr/share/usergrid/cql/update_usergrid_applications.cql
+sed -i.bak "s/REPLICATION_FACTOR/${CASSANDRA_REPLICATION_FACTOR}/g" /usr/share/usergrid/cql/update_usergrid_applications.cql
/usr/bin/cassandra-cli -h ${CASSHOST} -f /usr/share/usergrid/cql/update_usergrid_applications.cql
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3930e85f/stack/awscluster/src/main/groovy/configure_usergrid.groovy
----------------------------------------------------------------------
diff --git a/stack/awscluster/src/main/groovy/configure_usergrid.groovy b/stack/awscluster/src/main/groovy/configure_usergrid.groovy
index 128d271..21e1bad 100644
--- a/stack/awscluster/src/main/groovy/configure_usergrid.groovy
+++ b/stack/awscluster/src/main/groovy/configure_usergrid.groovy
@@ -17,9 +17,9 @@
*/
-//
-// configure_usergrid.groovy
-//
+//
+// configure_usergrid.groovy
+//
// Emits usergrid properties file based on environment and Cassandra node registry in SimpleDB
//
import com.amazonaws.auth.*
@@ -36,20 +36,23 @@ String domain = stackName
String hostName = (String)System.getenv().get("PUBLIC_HOSTNAME")
def replFactor = System.getenv().get("CASSANDRA_REPLICATION_FACTOR")
def clusterName = System.getenv().get("CASSANDRA_CLUSTER_NAME")
+def readConsistencyLevel = System.getenv().get("CASSANDRA_READ_CONSISTENCY")
+def writeConsistencyLevel = System.getenv().get("CASSANDRA_WRITE_CONSISTENCY")
def superUserEmail = System.getenv().get("SUPER_USER_EMAIL")
def testAdminUserEmail = System.getenv().get("TEST_ADMIN_USER_EMAIL")
def numEsNodes = Integer.parseInt(System.getenv().get("ES_NUM_SERVERS"))
//Override number of shards. Set it to 2x the cluster size
-def esShards = numEsNodes*4;
+def esShards = numEsNodes*2;
//This gives us 3 copies, which means we'll have a quorum with primary + 1 replica
-def esReplicas = 2;
+def esReplicas = 1;
def cassThreads = System.getenv().get("TOMCAT_THREADS")
-def hystrixThreads = Integer.parseInt(cassThreads) / 100
+//temporarily set to equal since we now have a sane tomcat thread calculation
+def hystrixThreads = cassThreads
//if we end in -1, we remove it
def ec2Region = System.getenv().get("EC2_REGION")
@@ -88,7 +91,7 @@ for (item in selectResult) {
def usergridConfig = """
######################################################
-# Minimal Usergrid configuration properties for local Tomcat and Cassandra
+# Minimal Usergrid configuration properties for local Tomcat and Cassandra
cassandra.url=${cassandras}
cassandra.cluster=${clusterName}
@@ -99,6 +102,10 @@ cassandra.timeout=5000
cassandra.connections=${cassThreads}
hystrix.threadpool.graph_user.coreSize=${hystrixThreads}
hystrix.threadpool.graph_async.coreSize=${hystrixThreads}
+usergrid.read.cl=${readConsistencyLevel}
+usergrid.write.cl=${writeConsistencyLevel}
+
+
elasticsearch.cluster_name=${clusterName}
elasticsearch.index_prefix=usergrid
@@ -108,7 +115,7 @@ elasticsearch.number_shards=${esShards}
elasticsearch.number_replicas=${esReplicas}
######################################################
-# Custom mail transport
+# Custom mail transport
mail.transport.protocol=smtp
mail.smtp.host=localhost
@@ -186,4 +193,4 @@ usergrid.queue.region=${ec2Region}
"""
-println usergridConfig
+println usergridConfig
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3930e85f/stack/awscluster/ugcluster-cf.json
----------------------------------------------------------------------
diff --git a/stack/awscluster/ugcluster-cf.json b/stack/awscluster/ugcluster-cf.json
index 02058e5..3396597 100644
--- a/stack/awscluster/ugcluster-cf.json
+++ b/stack/awscluster/ugcluster-cf.json
@@ -78,9 +78,20 @@
"CassReplicationFactor": {
"Description": "Cassandra replication factor",
"Type": "Number",
- "Default": "3",
+ "Default": "2",
"MinValue": "1"
},
+
+ "CassReadConsistency": {
+ "Description": "Cassandra replication factor for Astyanax",
+ "Type": "String",
+ "Default": "CL_ONE"
+ },
+ "CassWriteConsistency": {
+ "Description": "Cassandra replication factor for Astyanax",
+ "Type": "String",
+ "Default": "CL_ONE"
+ },
"ESClusterName": {
"Description": "Name to be used for Elasticsearch cluster.",
"Type": "String",
@@ -1149,6 +1160,11 @@
"\n",
"export CASSANDRA_REPLICATION_FACTOR=", { "Ref":"CassReplicationFactor" }, "\n",
"\n",
+ "export CASSANDRA_READ_CONSISTENCY=", { "Ref":"CassReadConsistency" }, "\n",
+ "\n",
+ "export CASSANDRA_WRITE_CONSISTENCY=", { "Ref":"CassWriteConsistency" }, "\n",
+ "\n",
+
"export ES_CLUSTER_NAME=", { "Ref":"ESClusterName" }, "\n",
"export ES_NUM_SERVERS=", { "Ref":"ESNumServers" }, "\n",
"\n",
[2/2] incubator-usergrid git commit: Merge branch 'two-dot-o-import'
of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into
two-dot-o-import
Posted by to...@apache.org.
Merge branch 'two-dot-o-import' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into two-dot-o-import
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/0fb13ee0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/0fb13ee0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/0fb13ee0
Branch: refs/heads/two-dot-o-import
Commit: 0fb13ee0be102bc4094d9605c0f677d11b5c0d52
Parents: 3930e85 7d27c23
Author: Todd Nine <tn...@apigee.com>
Authored: Thu Feb 5 18:48:35 2015 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Thu Feb 5 18:48:35 2015 -0700
----------------------------------------------------------------------
.../management/export/ExportServiceImpl.java | 15 +-
.../management/export/ExportServiceIT.java | 360 +++++++++----------
.../management/export/MockS3ExportImpl.java | 4 +-
.../management/importer/ImportCollectionIT.java | 279 ++++++++------
4 files changed, 353 insertions(+), 305 deletions(-)
----------------------------------------------------------------------