You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by jz...@apache.org on 2017/07/18 12:04:23 UTC
[opennlp-sandbox] branch master updated: OPENNLP-1111: Adding
initial EC2 scripts for testing.
This is an automated email from the ASF dual-hosted git repository.
jzemerick pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/opennlp-sandbox.git
The following commit(s) were added to refs/heads/master by this push:
new cba153e OPENNLP-1111: Adding initial EC2 scripts for testing.
cba153e is described below
commit cba153eecb5344db91402203ba787c369f6bf8f4
Author: jzonthemtn <jz...@apache.org>
AuthorDate: Thu Jul 13 15:07:28 2017 -0400
OPENNLP-1111: Adding initial EC2 scripts for testing.
---
aws-ec2-testing-scripts/README.md | 25 +++
aws-ec2-testing-scripts/cf-template.json | 199 +++++++++++++++++++++++
aws-ec2-testing-scripts/notify.sh | 36 ++++
aws-ec2-testing-scripts/run-eval-tests.sh | 19 +++
aws-ec2-testing-scripts/run-high-memory-tests.sh | 19 +++
5 files changed, 298 insertions(+)
diff --git a/aws-ec2-testing-scripts/README.md b/aws-ec2-testing-scripts/README.md
new file mode 100644
index 0000000..16b7752
--- /dev/null
+++ b/aws-ec2-testing-scripts/README.md
@@ -0,0 +1,25 @@
+# OpenNLP Testing Scripts
+
+These are scripts useful when testing OpenNLP builds on EC2.
+
+## Directory Structure
+
+These scripts are written expecting the following directory structure:
+
+* `/opt/` - Contains these scripts.
+* `/opt/opennlp` - Contains the OpenNLP code as cloned from https://github.com/apache/opennlp.
+* `/opt/opennlp-data` - Contains the data required for some of the OpenNLP tests. Contact dev@opennlp.apache.org for information on this data.
+
+## EC2 Instance Requirements
+
+* The instance must have the AWS CLI installed.
+* The scripts use SNS to send notifications so the instance must have permissions to publish SNS messages through either an instance role or via access/secret keys configured in the AWS CLI.
+* You must have an existing SNS topic configured to publish messages to and you must set the ARN in the `notify.sh` script.
+
+## Notifications and Results
+
+You can configure the subject, message, and destination (topic ARN) in the `notify.sh` script. The build log will be too large (>256KB) for sending in the SNS message. You can modify the `notify.sh` script for uploading the build results to an S3 bucket.
+
+## CloudFormation Template
+
+The CloudFormation template can help with creating the instance.
diff --git a/aws-ec2-testing-scripts/cf-template.json b/aws-ec2-testing-scripts/cf-template.json
new file mode 100644
index 0000000..45fe14f
--- /dev/null
+++ b/aws-ec2-testing-scripts/cf-template.json
@@ -0,0 +1,199 @@
+{
+ "AWSTemplateFormatVersion": "2010-09-09",
+ "Description": "Stack for running OpenNLP testing.",
+ "Parameters": {
+ "InstanceType": {
+ "Description": "EC2 instance type.",
+ "Type": "String",
+ "Default": "r4.xlarge"
+ },
+ "Image": {
+ "Description": "The base AMI.",
+ "Type": "String",
+ "Default": "ami-80861296"
+ },
+ "KeyName": {
+ "Description": "An existing EC2 keypair.",
+ "Type": "AWS::EC2::KeyPair::KeyName",
+ "ConstraintDescription": "Must be the name of an existing EC2 keypair."
+ }
+ },
+ "Resources": {
+ "VPC": {
+ "Type": "AWS::EC2::VPC",
+ "Properties": {
+ "CidrBlock": "10.0.0.0/16",
+ "Tags": [
+ {
+ "Key": "Application",
+ "Value": {
+ "Ref": "AWS::StackId"
+ }
+ }
+ ]
+ }
+ },
+ "Subnet": {
+ "Type": "AWS::EC2::Subnet",
+ "Properties": {
+ "VpcId": {
+ "Ref": "VPC"
+ },
+ "CidrBlock": "10.0.0.0/24",
+ "Tags": [
+ {
+ "Key": "Application",
+ "Value": {
+ "Ref": "AWS::StackId"
+ }
+ }
+ ]
+ }
+ },
+ "InternetGateway": {
+ "Type": "AWS::EC2::InternetGateway",
+ "Properties": {
+ "Tags": [
+ {
+ "Key": "Application",
+ "Value": {
+ "Ref": "AWS::StackId"
+ }
+ }
+ ]
+ }
+ },
+ "AttachGateway": {
+ "Type": "AWS::EC2::VPCGatewayAttachment",
+ "Properties": {
+ "VpcId": {
+ "Ref": "VPC"
+ },
+ "InternetGatewayId": {
+ "Ref": "InternetGateway"
+ }
+ }
+ },
+ "RouteTable": {
+ "Type": "AWS::EC2::RouteTable",
+ "Properties": {
+ "VpcId": {
+ "Ref": "VPC"
+ },
+ "Tags": [
+ {
+ "Key": "Application",
+ "Value": {
+ "Ref": "AWS::StackId"
+ }
+ }
+ ]
+ }
+ },
+ "Route": {
+ "Type": "AWS::EC2::Route",
+ "DependsOn": "AttachGateway",
+ "Properties": {
+ "RouteTableId": {
+ "Ref": "RouteTable"
+ },
+ "DestinationCidrBlock": "0.0.0.0/0",
+ "GatewayId": {
+ "Ref": "InternetGateway"
+ }
+ }
+ },
+ "SubnetRouteTableAssociation": {
+ "Type": "AWS::EC2::SubnetRouteTableAssociation",
+ "Properties": {
+ "SubnetId": {
+ "Ref": "Subnet"
+ },
+ "RouteTableId": {
+ "Ref": "RouteTable"
+ }
+ }
+ },
+ "NetworkAcl": {
+ "Type": "AWS::EC2::NetworkAcl",
+ "Properties": {
+ "VpcId": {
+ "Ref": "VPC"
+ },
+ "Tags": [
+ {
+ "Key": "Application",
+ "Value": {
+ "Ref": "AWS::StackId"
+ }
+ }
+ ]
+ }
+ },
+ "InstanceSecurityGroup": {
+ "Type": "AWS::EC2::SecurityGroup",
+ "Properties": {
+ "VpcId": {
+ "Ref": "VPC"
+ },
+ "GroupDescription": "Enable SSH access via port 22",
+ "SecurityGroupIngress": [
+ {
+ "IpProtocol": "tcp",
+ "FromPort": "22",
+ "ToPort": "22",
+ "CidrIp": "0.0.0.0/0"
+ }
+ ]
+ }
+ },
+ "OpenNLPInstance": {
+ "Type": "AWS::EC2::Instance",
+ "DependsOn": "AttachGateway",
+ "Properties": {
+ "ImageId": {
+ "Ref": "Image"
+ },
+ "InstanceType": {
+ "Ref": "InstanceType"
+ },
+ "KeyName": {
+ "Ref": "KeyName"
+ },
+ "Tags": [
+ {
+ "Key": "Application",
+ "Value": {
+ "Ref": "AWS::StackId"
+ }
+ },
+ {
+ "Key": "Name",
+ "Value": "OpenNLP Testing"
+ }
+ ],
+ "UserData": {
+ "Fn::Base64": {
+ "Fn::Join": [
+ "",
+ [
+ "#!/bin/bash -xe\n",
+ "apt-get update && sudo apt-get -y dist-upgrade\n",
+ "apt-get install -y openjdk-8-jdk git maven awscli\n",
+ "# Get the scripts\n",
+ "git clone https://github.com/apache/opennlp-sandbox.git\n",
+ "mv opennlp-sandbox/aws-ec2-testing-scripts/* /opt/\n",
+ "# Get OpenNLP\n",
+ "git clone https://github.com/apache/opennlp.git\n",
+ "mv opennlp /opt/\n",
+ "mkdir /opt/opennlp-data\n",
+ "aws s3 cp s3://bucket/opennlp-data /opt/opennlp-data --recursive\n"
+ ]
+ ]
+ }
+ }
+ }
+ }
+ },
+ "Outputs": {}
+}
diff --git a/aws-ec2-testing-scripts/notify.sh b/aws-ec2-testing-scripts/notify.sh
new file mode 100755
index 0000000..3a853e8
--- /dev/null
+++ b/aws-ec2-testing-scripts/notify.sh
@@ -0,0 +1,36 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Configure these values as desired.
+
+# Notifications via SNS.
+TOPIC_ARN="arn:aws:sns:us-east-1:XXXXXXXXXXXX:opennlp-notification"
+SUBJECT="OpenNLP Notification"
+
+# Upload of build output to S3. (Uncomment below to enable.)
+BUCKET=""
+
+# Received from the training scripts.
+MESSAGE=$1
+ACTION=$2
+
+# Publish the message to SNS.
+aws sns publish --topic-arn "$TOPIC_ARN" --message "$MESSAGE" --subject "$SUBJECT"
+
+# Upload the build output to S3.
+TIMESTAMP=`date +"%T"`
+#aws s3 cp nohup.out s3://$BUCKET/$2-output-$TIMESTAMP.txt
diff --git a/aws-ec2-testing-scripts/run-eval-tests.sh b/aws-ec2-testing-scripts/run-eval-tests.sh
new file mode 100755
index 0000000..b98bd1e
--- /dev/null
+++ b/aws-ec2-testing-scripts/run-eval-tests.sh
@@ -0,0 +1,19 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+rm -f ./nohup.out
+nohup sh -c 'cd opennlp && mvn clean install -Peval-tests -DOPENNLP_DATA_DIR=/opt/opennlp-data/ && /opt/notify.sh "eval-tests complete" "eval-tests"' &
diff --git a/aws-ec2-testing-scripts/run-high-memory-tests.sh b/aws-ec2-testing-scripts/run-high-memory-tests.sh
new file mode 100755
index 0000000..3891318
--- /dev/null
+++ b/aws-ec2-testing-scripts/run-high-memory-tests.sh
@@ -0,0 +1,19 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+rm -f nohup.out
+nohup sh -c 'cd opennlp && mvn clean install -Phigh-memory-tests -DOPENNLP_DATA_DIR=/opt/opennlp-data/ && /opt/notify.sh "High memory tests complete" "high-memory-tests"' &
--
To stop receiving notification emails like this one, please contact
['"commits@opennlp.apache.org" <co...@opennlp.apache.org>'].