You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by to...@apache.org on 2014/10/14 23:00:35 UTC
[11/23] git commit: separated ES from C* in cloudformation
template/scripts
separated ES from C* in cloudformation template/scripts
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/112661a0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/112661a0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/112661a0
Branch: refs/heads/cloudformation-update
Commit: 112661a0799334c792641fa917b77d205ee8c9a6
Parents: 43acfad
Author: amuramoto <am...@apigee.com>
Authored: Fri Oct 10 12:35:45 2014 -0700
Committer: amuramoto <am...@apigee.com>
Committed: Fri Oct 10 12:35:45 2014 -0700
----------------------------------------------------------------------
.../main/dist/init_instance/init_db_server.sh | 4 -
.../main/dist/init_instance/init_es_server.sh | 61 +++++
stack/awscluster/ugcluster-cf.json | 243 ++++++++++++++++++-
3 files changed, 303 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/112661a0/stack/awscluster/src/main/dist/init_instance/init_db_server.sh
----------------------------------------------------------------------
diff --git a/stack/awscluster/src/main/dist/init_instance/init_db_server.sh b/stack/awscluster/src/main/dist/init_instance/init_db_server.sh
index b9c4f56..a37bb82 100644
--- a/stack/awscluster/src/main/dist/init_instance/init_db_server.sh
+++ b/stack/awscluster/src/main/dist/init_instance/init_db_server.sh
@@ -59,10 +59,6 @@ cd /usr/share/usergrid/init_instance
cd /usr/share/usergrid/init_instance
./install_opscenter_agent.sh
-# Install and start ElasticSearch
-cd /usr/share/usergrid/init_instance
-./install_elasticsearch.sh
-
# Use the CQL to crate the keyspaces
cd /usr/share/usergrid/init_instance
./create_keyspaces.sh
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/112661a0/stack/awscluster/src/main/dist/init_instance/init_es_server.sh
----------------------------------------------------------------------
diff --git a/stack/awscluster/src/main/dist/init_instance/init_es_server.sh b/stack/awscluster/src/main/dist/init_instance/init_es_server.sh
new file mode 100644
index 0000000..c187cb8
--- /dev/null
+++ b/stack/awscluster/src/main/dist/init_instance/init_es_server.sh
@@ -0,0 +1,61 @@
+#!/bin/bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. The ASF licenses this file to You
+# under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License. For additional information regarding
+# copyright in this work, please see the NOTICE file in the top level
+# directory of this distribution.
+#
+
+echo "${HOSTNAME}" > /etc/hostname
+echo "127.0.0.1 ${HOSTNAME}" >> /etc/hosts
+hostname `cat /etc/hostname`
+
+echo "US/Eastern" > /etc/timezone
+dpkg-reconfigure -f noninteractive tzdata
+
+# Build environment for scripts
+. /etc/profile.d/aws-credentials.sh
+. /etc/profile.d/usergrid-env.sh
+
+cd /usr/share/usergrid/init_instance
+./create_raid0.sh
+
+# Install the easy stuff
+PKGS="ntp unzip groovy curl"
+apt-get update
+apt-get -y --force-yes install ${PKGS}
+/etc/init.d/tomcat7 stop
+
+# Install AWS Java SDK and get it into the Groovy classpath
+curl http://sdk-for-java.amazonwebservices.com/latest/aws-java-sdk.zip > /tmp/aws-sdk-java.zip
+cd /usr/share/
+unzip /tmp/aws-sdk-java.zip
+mkdir -p /home/ubuntu/.groovy/lib
+cp /usr/share/aws-java-sdk-*/third-party/*/*.jar /home/ubuntu/.groovy/lib
+cp /usr/share/aws-java-sdk-*/lib/* /home/ubuntu/.groovy/lib
+# except for evil stax
+rm /home/ubuntu/.groovy/lib/stax*
+ln -s /home/ubuntu/.groovy /root/.groovy
+
+cd /usr/share/usergrid/init_instance
+./install_oraclejdk.sh
+
+# Install and start ElasticSearch
+cd /usr/share/usergrid/init_instance
+./install_elasticsearch.sh
+
+# tag last so we can see in the console that the script ran to completion
+cd /usr/share/usergrid/scripts
+groovy tag_instance.groovy
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/112661a0/stack/awscluster/ugcluster-cf.json
----------------------------------------------------------------------
diff --git a/stack/awscluster/ugcluster-cf.json b/stack/awscluster/ugcluster-cf.json
index 67c081a..31950a2 100644
--- a/stack/awscluster/ugcluster-cf.json
+++ b/stack/awscluster/ugcluster-cf.json
@@ -84,6 +84,28 @@
"Default": "3",
"MinValue": "1"
},
+ "ESNumServers": {
+ "Description": "Number of ES servers to start.",
+ "Type": "Number",
+ "Default": "3",
+ "MinValue": "1"
+ },
+ "ESInstanceType": {
+ "Description": "Instance type for ES servers",
+ "Type": "String",
+ "Default": "c3.2xlarge",
+ "AllowedValues": [
+ "m1.small",
+ "m1.medium",
+ "m1.large",
+ "m1.xlarge",
+ "m3.xlarge",
+ "m3.2xlarge",
+ "c3.2xlarge",
+ "c3.4xlarge"
+ ],
+ "ConstraintDescription": "must be valid instance type."
+ },
"SuperUserEmail": {
"Description": "Email for superuser user",
"Type": "String",
@@ -429,6 +451,34 @@
}
}
},
+ "ESUser": {
+ "Type": "AWS::IAM::User",
+ "Properties": {
+ "Path": "/",
+ "Policies": [
+ {
+ "PolicyName": "root",
+ "PolicyDocument": {
+ "Statement": [
+ {
+ "Effect": "Allow",
+ "Action": "*",
+ "Resource": "*"
+ }
+ ]
+ }
+ }
+ ]
+ }
+ },
+ "ESKey": {
+ "Type": "AWS::IAM::AccessKey",
+ "Properties": {
+ "UserName": {
+ "Ref": "CassUser"
+ }
+ }
+ },
"CassAutoScalingLaunchConfiguration":{
"Type":"AWS::AutoScaling::LaunchConfiguration",
"Properties":{
@@ -588,6 +638,155 @@
}
}
},
+ "ESAutoScalingLaunchConfiguration":{
+ "Type":"AWS::AutoScaling::LaunchConfiguration",
+ "Properties":{
+ "UserData":{
+ "Fn::Base64":{
+ "Fn::Join":[
+ "",
+ [
+ "#!/bin/bash -ex\n",
+ "# ES NODE STARTUP \n",
+ "exec >/var/log/usergrid-bootstrap.log 2>&1\n",
+ "\n",
+ "mkdir -p /usr/share/usergrid\n",
+ "\n",
+ "# create script that sets our environment variables\n",
+ "cat >/etc/profile.d/usergrid-env.sh <<EOF\n",
+ "alias sudo='sudo -E'\n", "\n",
+ "export TYPE=es\n",
+ "export STACK_NAME=", { "Ref":"AWS::StackName" }, "\n", "\n",
+ "export PUBLIC_HOSTNAME=`(curl -s http://169.254.169.254/latest/meta-data/public-hostname)`\n",
+ "export INTERNAL_HOSTNAME=`(curl http://169.254.169.254/latest/meta-data/local-ipv4)`\n",
+ "\n",
+ "export EC2_INSTANCE_ID=`ec2metadata --instance-id`\n",
+ "export EC2_REGION=", { "Ref":"AWS::Region" },
+ "\n",
+ "export EC2_URL=https://ec2.amazonaws.com/\n",
+ "\n",
+ "export ES_SECURITY_GROUP_NAME=", { "Ref":"ESSecurityGroup" }, "\n",
+ "\n",
+ "\n",
+ "export ES_NUM_SERVERS=", { "Ref":"ESNumServers" }, "\n",
+ "export GRAPHITE_NUM_SERVERS=", { "Ref":"GraphiteNumServers" }, "\n",
+ "\n",
+ "export RELEASE_BUCKET=", { "Ref":"ReleaseBucket" }, "\n",
+ "\n",
+ "EOF\n",
+ "\n",
+ "# put AWS creds in environment\n",
+ "cat >/etc/profile.d/aws-credentials.sh <<EOF\n",
+ "export AWS_ACCESS_KEY=", { "Ref":"ESKey" }, "\n",
+ "export AWS_SECRET_KEY=", { "Fn::GetAtt":[ "ESKey", "SecretAccessKey" ] }, "\n",
+ "EOF\n",
+ "\n",
+ "# setup s3cmd (will be installed by init script) \n",
+ "cat >/etc/s3cfg <<EOF\n",
+ "access_key=", { "Ref":"ESKey" }, "\n",
+ "secret_key=", { "Fn::GetAtt":[ "ESKey", "SecretAccessKey" ] },
+ "\n",
+ "EOF\n",
+ "chmod 644 /etc/s3cfg\n",
+ "ln -s /etc/s3cfg ~ubuntu/.s3cfg\n",
+ "ln -s /etc/s3cfg ~root/.s3cfg\n",
+ "\n",
+ "# download usergrid and init script bundle from S3\n",
+ "wget -O- -q http://s3tools.org/repo/deb-all/stable/s3tools.key | apt-key add -\n",
+ "wget -O/etc/apt/sources.list.d/s3tools.list http://s3tools.org/repo/deb-all/stable/s3tools.list\n",
+ "apt-get update\n",
+ "apt-get -y install s3cmd\n",
+ "cd /usr/share/usergrid\n",
+ "s3cmd --config=/etc/s3cfg get s3://", { "Ref":"ReleaseBucket" }, "/awscluster-1.0-SNAPSHOT-any.tar.gz\n",
+ "tar xvf awscluster-1.0-SNAPSHOT-any.tar.gz\n",
+ "rm -fr awscluster-1.0-SNAPSHOT-any.tar.gz\n",
+ "chmod 755 ./init_instance/*.sh\n",
+ "cd ./init_instance\n",
+ "# init as an ES node \n",
+ "sh ./init_es_server.sh\n"
+ ]
+ ]
+ }
+ },
+ "KeyName": {
+ "Ref": "KeyPair"
+ },
+ "ImageId": {
+ "Fn::FindInMap": [
+ "AWSRegionArch2AMI",
+ {
+ "Ref": "AWS::Region"
+ },
+ {
+ "Fn::FindInMap": [
+ "AWSInstanceType2Arch",
+ {
+ "Ref": "CassInstanceType"
+ },
+ "Arch"
+ ]
+ }
+ ]
+ },
+ "InstanceType": {
+ "Ref": "ESInstanceType"
+ },
+ "IamInstanceProfile": {
+ "Ref": "RootInstanceProfile"
+ },
+ "SecurityGroups": [
+ {
+ "Ref": "ESSecurityGroup"
+ }
+ ]
+ }
+ },
+ "ESAutoScalingGroup": {
+ "Type": "AWS::AutoScaling::AutoScalingGroup",
+ "Version": "2009-05-15",
+ "Properties": {
+ "AvailabilityZones": [
+ {
+ "Fn::FindInMap": [
+ "TwoAZs",
+ {
+ "Ref": "AWS::Region"
+ },
+ "AZ1"
+ ]
+ },
+ {
+ "Fn::FindInMap": [
+ "TwoAZs",
+ {
+ "Ref": "AWS::Region"
+ },
+ "AZ2"
+ ]
+ }
+ ],
+ "LaunchConfigurationName": {
+ "Ref": "ESAutoScalingLaunchConfiguration"
+ },
+ "MinSize": {
+ "Ref": "ESNumServers"
+ },
+ "MaxSize": {
+ "Ref": "ESNumServers"
+ },
+ "NotificationConfiguration": {
+ "TopicARN": {
+ "Ref": "NotificationTopic"
+ },
+ "NotificationTypes": [
+ "autoscaling:EC2_INSTANCE_LAUNCH",
+ "autoscaling:EC2_INSTANCE_LAUNCH_ERROR",
+ "autoscaling:EC2_INSTANCE_TERMINATE",
+ "autoscaling:EC2_INSTANCE_TERMINATE_ERROR"
+ ]
+ }
+ }
+ },
"RestAutoScalingLaunchConfiguration":{
"Type":"AWS::AutoScaling::LaunchConfiguration",
"Properties":{
@@ -980,6 +1179,34 @@
]
}
},
+ "ESSecurityGroup": {
+ "Type": "AWS::EC2::SecurityGroup",
+ "Properties": {
+ "GroupDescription": "ElasticSearch Machines",
+ "SecurityGroupIngress": [
+ {
+ "IpProtocol": "tcp",
+ "FromPort": "22",
+ "ToPort": "22",
+ "CidrIp": "0.0.0.0/0"
+ }
+ ]
+ }
+ },
+ "AllowCassToESOnAllPorts": {
+ "Type": "AWS::EC2::SecurityGroupIngress",
+ "Properties": {
+ "GroupName": {
+ "Ref": "ESSecurityGroup"
+ },
+ "IpProtocol": "tcp",
+ "FromPort": "1",
+ "ToPort": "65535",
+ "SourceSecurityGroupName": {
+ "Ref": "CassSecurityGroup"
+ }
+ }
+ },
"CassSecurityGroup": {
"Type": "AWS::EC2::SecurityGroup",
"Properties": {
@@ -993,7 +1220,7 @@
}
]
}
- },
+ },
"AllowCassToCassOnAllPorts": {
"Type": "AWS::EC2::SecurityGroupIngress",
"Properties": {
@@ -1102,6 +1329,20 @@
]
}
},
+ "AllowESToGraphiteAllPorts": {
+ "Type": "AWS::EC2::SecurityGroupIngress",
+ "Properties": {
+ "GroupName": {
+ "Ref": "GraphiteSecurityGroup"
+ },
+ "IpProtocol": "tcp",
+ "FromPort": "0",
+ "ToPort": "65535",
+ "SourceSecurityGroupName": {
+ "Ref": "ESSecurityGroup"
+ }
+ }
+ },
"AllowCassToGraphiteAllPorts": {
"Type": "AWS::EC2::SecurityGroupIngress",
"Properties": {