You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sf...@apache.org on 2014/12/03 18:58:19 UTC
[04/16] incubator-usergrid git commit: Updated NodeRegistry to use
only tags instead of external simple db
Updated NodeRegistry to use only tags instead of external simple db
Upgraded AWS client
Removed unused registry list groovy script
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/b12130f0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/b12130f0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/b12130f0
Branch: refs/heads/two-dot-o
Commit: b12130f06d503f8c5a3fc29d6049a4df89eb5a7e
Parents: 1f75549
Author: Todd Nine <tn...@apigee.com>
Authored: Tue Nov 25 17:38:42 2014 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Tue Nov 25 17:38:42 2014 -0700
----------------------------------------------------------------------
stack/awscluster/gatling-cluster-cf.json | 86 +++++++---
stack/awscluster/pom.xml | 2 +-
.../src/main/groovy/NodeRegistry.groovy | 165 ++++++++++++-------
.../src/main/groovy/registry_list.groovy | 36 ----
4 files changed, 168 insertions(+), 121 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b12130f0/stack/awscluster/gatling-cluster-cf.json
----------------------------------------------------------------------
diff --git a/stack/awscluster/gatling-cluster-cf.json b/stack/awscluster/gatling-cluster-cf.json
index ff41f14..b9f9752 100644
--- a/stack/awscluster/gatling-cluster-cf.json
+++ b/stack/awscluster/gatling-cluster-cf.json
@@ -1,53 +1,89 @@
{
"AWSTemplateFormatVersion" : "2010-09-09",
"Description" : "Gatling cluster",
-
- "Parameters" : {
+ "Parameters": {
"InstanceCount": {
- "Description" : "The number of gatling instances to create",
+ "Description": "The number of gatling instances to create",
"Type": "Number",
"Default": "1",
"MinValue": "1"
},
-
"KeyName": {
- "Description" : "Name of an existing EC2 KeyPair to enable SSH access to the instance",
+ "Description": "Name of an existing EC2 KeyPair to enable SSH access to the instance",
"Type": "AWS::EC2::KeyPair::KeyName",
- "Default" : "jenkins-east",
- "ConstraintDescription" : "must be the name of an existing EC2 KeyPair."
+ "Default": "jenkins-east",
+ "ConstraintDescription": "must be the name of an existing EC2 KeyPair."
},
-
- "InstanceType" : {
- "Description" : "WebServer EC2 instance type",
- "Type" : "String",
- "Default" : "m1.xlarge",
- "AllowedValues" : [ "t1.micro", "t2.micro", "t2.small", "t2.medium", "m1.small", "m1.medium", "m1.large", "m1.xlarge", "m2.xlarge", "m2.2xlarge", "m2.4xlarge", "m3.medium", "m3.large", "m3.xlarge", "m3.2xlarge", "c1.medium", "c1.xlarge", "c3.large", "c3.xlarge", "c3.2xlarge", "c3.4xlarge", "c3.8xlarge", "g2.2xlarge", "r3.large", "r3.xlarge", "r3.2xlarge", "r3.4xlarge", "r3.8xlarge", "i2.xlarge", "i2.2xlarge", "i2.4xlarge", "i2.8xlarge", "hi1.4xlarge", "hs1.8xlarge", "cr1.8xlarge", "cc2.8xlarge", "cg1.4xlarge"]
- ,
- "ConstraintDescription" : "must be a valid EC2 instance type."
+ "InstanceType": {
+ "Description": "WebServer EC2 instance type",
+ "Type": "String",
+ "Default": "m1.xlarge",
+ "AllowedValues": [
+ "t1.micro",
+ "t2.micro",
+ "t2.small",
+ "t2.medium",
+ "m1.small",
+ "m1.medium",
+ "m1.large",
+ "m1.xlarge",
+ "m2.xlarge",
+ "m2.2xlarge",
+ "m2.4xlarge",
+ "m3.medium",
+ "m3.large",
+ "m3.xlarge",
+ "m3.2xlarge",
+ "c1.medium",
+ "c1.xlarge",
+ "c3.large",
+ "c3.xlarge",
+ "c3.2xlarge",
+ "c3.4xlarge",
+ "c3.8xlarge",
+ "g2.2xlarge",
+ "r3.large",
+ "r3.xlarge",
+ "r3.2xlarge",
+ "r3.4xlarge",
+ "r3.8xlarge",
+ "i2.xlarge",
+ "i2.2xlarge",
+ "i2.4xlarge",
+ "i2.8xlarge",
+ "hi1.4xlarge",
+ "hs1.8xlarge",
+ "cr1.8xlarge",
+ "cc2.8xlarge",
+ "cg1.4xlarge"
+ ],
+ "ConstraintDescription": "must be a valid EC2 instance type."
},
-
"OperatorEMail": {
"Description": "EMail address to notify if there are any scaling operations",
"Type": "String",
"AllowedPattern": "([a-zA-Z0-9_\\-\\.]+)@((\\[[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.)|(([a-zA-Z0-9\\-]+\\.)+))([a-zA-Z]{2,4}|[0-9]{1,3})(\\]?)",
"ConstraintDescription": "must be a valid email address.",
- "Default":"rbridges@apigee.com"
+ "Default": "rbridges@apigee.com"
},
-
"ReleaseBucket": {
"Description": "S3 Bucket where Usergrid assembly is to be found.",
"Type": "String",
"Default": "usergrid-jenkins-builds"
},
-
- "SSHLocation" : {
- "Description" : "The IP address range that can be used to SSH to the EC2 instances",
+ "SSHLocation": {
+ "Description": "The IP address range that can be used to SSH to the EC2 instances",
"Type": "String",
"MinLength": "9",
"MaxLength": "18",
"Default": "0.0.0.0/0",
"AllowedPattern": "(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})/(\\d{1,2})",
"ConstraintDescription": "must be a valid IP CIDR range of the form x.x.x.x/x."
+ },
+ "Branch": {
+ "Description": "The branch of usergrid to check out",
+ "Type": "String",
+ "Default": "two-dot-o"
}
},
@@ -231,15 +267,11 @@
"apt-add-repository ppa:awstools-dev/awstools --yes\n",
"apt-get update\n",
"echo oracle-java7-installer shared/accepted-oracle-license-v1-1 select true | /usr/bin/debconf-set-selections\n",
- "apt-get install oracle-java7-installer oracle-java7-set-default expect expect-dev git ec2-api-tools -y --force-yes\n",
+ "apt-get install oracle-java7-installer oracle-java7-set-default expect expect-dev git ec2-api-tools maven -y --force-yes\n",
"update-java-alternatives -s java-7-oracle\n",
- "mkdir -p /usr/local/apache-maven\n",
- "cd /usr/local/apache-maven\n",
- "wget http://apache.mirrors.timporter.net/maven/maven-3/3.1.1/binaries/apache-maven-3.1.1-bin.tar.gz\n",
- "tar -xzvf apache-maven-3.1.1-bin.tar.gz\n",
"cd ~ubuntu\n",
- "git clone https://github.com/apache/incubator-usergrid.git usergrid\n",
+ "git clone -b ",{ "Ref":"Branch" } , " https://github.com/apache/incubator-usergrid.git usergrid\n",
"chown -R ubuntu:ubuntu usergrid\n",
"ec2-create-tags $(curl -k http://169.254.169.254/latest/meta-data/instance-id) --aws-access-key ",{ "Ref":"GatlingKey" }, " --aws-secret-key ", { "Fn::GetAtt":[ "GatlingKey", "SecretAccessKey" ] } ," --tag Name=", { "Ref":"AWS::StackName" }, "-Gatling\n"
]]}}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b12130f0/stack/awscluster/pom.xml
----------------------------------------------------------------------
diff --git a/stack/awscluster/pom.xml b/stack/awscluster/pom.xml
index 4a8f4d6..46157ab 100644
--- a/stack/awscluster/pom.xml
+++ b/stack/awscluster/pom.xml
@@ -215,7 +215,7 @@
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk</artifactId>
- <version>1.6.7</version>
+ <version>1.9.7</version>
</dependency>
<dependency>
<artifactId>groovy-all</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b12130f0/stack/awscluster/src/main/groovy/NodeRegistry.groovy
----------------------------------------------------------------------
diff --git a/stack/awscluster/src/main/groovy/NodeRegistry.groovy b/stack/awscluster/src/main/groovy/NodeRegistry.groovy
index 6291f93..277631b 100644
--- a/stack/awscluster/src/main/groovy/NodeRegistry.groovy
+++ b/stack/awscluster/src/main/groovy/NodeRegistry.groovy
@@ -20,34 +20,64 @@
* A utility class that search simple db for the node type provided and returns a list of hostnames as a string array
*/
import com.amazonaws.auth.BasicAWSCredentials
-import com.amazonaws.services.simpledb.AmazonSimpleDBClient
-import com.amazonaws.services.simpledb.model.*
+import com.amazonaws.regions.Region
+import com.amazonaws.regions.Regions
+import com.amazonaws.services.ec2.model.DescribeInstancesRequest
+import com.amazonaws.services.ec2.model.DescribeInstancesResult
+import com.amazonaws.services.ec2.model.DescribeTagsRequest
+import com.amazonaws.services.ec2.AmazonEC2Client
+import com.amazonaws.services.ec2.model.CreateTagsRequest
+import com.amazonaws.services.ec2.model.Filter
+import com.amazonaws.services.ec2.model.Instance
+import com.amazonaws.services.ec2.model.Tag
class NodeRegistry {
+
+ public static final String TAG_PREFIX = "tag:"
+ //taken from aws
+ public static final String STACK_NAME = "usergrid:stack-name";
+ public static final String NODE_TYPE = "usergrid:node_type";
+
private String accessKey = (String) System.getenv().get("AWS_ACCESS_KEY")
private String secretKey = (String) System.getenv().get("AWS_SECRET_KEY")
private String stackName = (String) System.getenv().get("STACK_NAME")
- private hostName = (String) System.getenv().get("PUBLIC_HOSTNAME")
+ private String instanceId = (String) System.getenv().get("EC2_INSTANCE_ID");
+ private String region = (String) System.getenv().get("EC2_REGION");
private String domain = stackName
- private def creds;
- private def sdbClient;
+ private BasicAWSCredentials creds;
+ private AmazonEC2Client ec2Client;
NodeRegistry() {
- while ( true ) {
- try {
- // creates domain or no-op if it already exists
- creds = new BasicAWSCredentials(accessKey, secretKey)
- sdbClient = new AmazonSimpleDBClient(creds)
- sdbClient.createDomain(new CreateDomainRequest(domain))
-
- } catch ( Exception e ) {
- continue
- }
- break
+
+ if (region == null) {
+ throw new IllegalArgumentException("EC2_REGION must be defined")
+ }
+
+ if (instanceId == null) {
+ throw new IllegalArgumentException("EC2_INSTANCE_ID must be defined")
+ }
+
+ if (stackName == null) {
+ throw new IllegalArgumentException("STACK_NAME must be defined")
+ }
+
+ if (accessKey == null) {
+ throw new IllegalArgumentException("AWS_ACCESS_KEY must be defined")
+ }
+
+ if (secretKey == null) {
+ throw new IllegalArgumentException("AWS_SECRET_KEY must be defined")
}
+
+ creds = new BasicAWSCredentials(accessKey, secretKey)
+ ec2Client = new AmazonEC2Client(creds)
+ def regionEnum = Regions.fromName(region);
+ ec2Client.setRegion(Region.getRegion(regionEnum))
+
+
}
/**
@@ -55,64 +85,85 @@ class NodeRegistry {
* @param defNodeType
*/
def searchNode(def nodeType) {
- //order by create time, if we have a conflict, then order by item name
- def selectResult = sdbClient.select(new SelectRequest((String) \
- "select * from `${domain}` where itemName() is not null AND createtime is not null AND nodetype = '${nodeType}' order by createtime"))
- def result = []
-
- for (item in selectResult.getItems()) {
- def hostname = item.getName()
- result.add(hostname)
+
+ def stackNameFilter = new Filter(TAG_PREFIX+STACK_NAME).withValues(stackName);
+ def nodeTypeFilter = new Filter(TAG_PREFIX+NODE_TYPE).withValues(nodeType);
+
+ def describeRequest = new DescribeInstancesRequest().withFilters(stackNameFilter, nodeTypeFilter);
+
+
+ def nodes = ec2Client.describeInstances(describeRequest)
+
+ //sort by created date
+ def servers = [];
+
+ for(reservation in nodes.getReservations()){
+
+ //TODO, add these to a list then sort them by date, then name
+ for(instance in reservation.getInstances()){
+
+ servers.add(new ServerEntry(instance.launchTime, instance.publicDnsName))
+ }
+
}
- return result
+
+ return createResults(servers);
}
- /**
- * Get the entire database back in raw form
- * @return
- */
- def selectAll() {
- def selectResult = sdbClient.select(new SelectRequest((String) "select * from `${domain}`")).getItems()
+ def createResults(def servers){
+
+
+ Collections.sort(servers);
+ def results = [];
- return selectResult;
+ for(server in servers){
+ results.add(server.publicIp)
+ }
+
+ return results;
}
+
+
/**
* Add the node to the database if it doesn't exist
*/
def addNode(def nodeType) {
- def gar = new GetAttributesRequest(domain, hostName);
- def response = sdbClient.getAttributes(gar);
- if (response.getAttributes().size() == 1) {
- println "Already registered"
- def attrs = response.getAttributes()
- for (att in attrs) {
- println("${hostName} -> ${att.getName()} : ${att.getValue()}")
- }
- return false;
-
- } else {
- println "Registering..."
- def stackAtt = new ReplaceableAttribute("nodetype", nodeType, true)
- def nowString = Calendar.getInstance(TimeZone.getTimeZone('UTC')).format("yyyy-MM-dd'T'HH:mm:ss.SSSZ")
- def createTime = new ReplaceableAttribute("createtime", nowString, true)
- def attrs = new ArrayList()
- attrs.add(stackAtt)
- attrs.add(createTime)
- def par = new PutAttributesRequest(domain, hostName, attrs)
- sdbClient.putAttributes(par);
- println "Registration done."
- return true;
- }
+ //add the node type
+ def tagRequest = new CreateTagsRequest().withTags(new Tag(NODE_TYPE, nodeType), new Tag(STACK_NAME, stackName)).withResources(instanceId)
+
+
+
+ ec2Client.createTags(tagRequest)
}
- def deleteRegistry(){
- sdbClient.deleteDomain(new DeleteDomainRequest(domain))
+
+ class ServerEntry implements Comparable<ServerEntry>{
+ private final Date launchDate;
+ private final String publicIp;
+
+ ServerEntry(final Date launchDate, final String publicIp) {
+ this.launchDate = launchDate
+ this.publicIp = publicIp
+ }
+
+ @Override
+ int compareTo(final ServerEntry o) {
+ if(launchDate.before(o.launchDate)){
+ -1;
+ }else if (launchDate.after(o.launchDate)){
+ return 1;
+ }
+
+ return publicIp.compareTo(o.publicIp);
+
+
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b12130f0/stack/awscluster/src/main/groovy/registry_list.groovy
----------------------------------------------------------------------
diff --git a/stack/awscluster/src/main/groovy/registry_list.groovy b/stack/awscluster/src/main/groovy/registry_list.groovy
deleted file mode 100644
index ca3bcda..0000000
--- a/stack/awscluster/src/main/groovy/registry_list.groovy
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. The ASF licenses this file to You
- * under the Apache License, Version 2.0 (the "License"); you may not
- * use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License. For additional information regarding
- * copyright in this work, please see the NOTICE file in the top level
- * directory of this distribution.
- */
-
-
-//
-// registry_list.groovy
-//
-// List contents of registry as debugging aid. Not used in any other scripts.
-//
-NodeRegistry registry = new NodeRegistry();
-
-
-
-
-for (item in registry.selectAll()) {
- for(att in item.getAttributes()){
- println "${item.getName()} -> ${att.getName()} : ${att.getValue()}"
- }
-
- println "\n"
-}