You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by wa...@apache.org on 2022/01/28 22:04:01 UTC
[pulsar] branch master updated: [Issue 5869][deploy] Bump AWS terraform deployment to Terraform 0.13 and AWS provider 3.3.0 (#7887)
This is an automated email from the ASF dual-hosted git repository.
wave pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new 57fcf0f [Issue 5869][deploy] Bump AWS terraform deployment to Terraform 0.13 and AWS provider 3.3.0 (#7887)
57fcf0f is described below
commit 57fcf0fb762ab53026f8e4c5bf9278798b31b8a8
Author: Koen Rutten <13...@users.noreply.github.com>
AuthorDate: Fri Jan 28 23:00:13 2022 +0100
[Issue 5869][deploy] Bump AWS terraform deployment to Terraform 0.13 and AWS provider 3.3.0 (#7887)
* Bump to Terraform 0.13 and AWS provider 3.3.0
* Update versions.tf
Unlock Terraform version
* Adjust the variable type to new Terraform version
* Ansible and terraform updates
Note that id_rsa is changed to pulsar_aws.
Documentation changes are required.
Co-authored-by: Koen Rutten <ko...@twilights.be>
Co-authored-by: Dave Fisher <da...@datastax.com>
Co-authored-by: Dave Fisher <da...@comcast.net>
---
deployment/terraform-ansible/aws/ansible.cfg | 2 +-
deployment/terraform-ansible/aws/instances.tf | 56 +++++++++++------------
deployment/terraform-ansible/aws/keys.tf | 4 +-
deployment/terraform-ansible/aws/network.tf | 50 ++++++++++----------
deployment/terraform-ansible/aws/provider.tf | 3 +-
deployment/terraform-ansible/aws/security.tf | 8 ++--
deployment/terraform-ansible/aws/setup-disk.yaml | 14 ++++--
deployment/terraform-ansible/aws/terraform.tfvars | 4 +-
deployment/terraform-ansible/aws/variables.tf | 2 +-
deployment/terraform-ansible/aws/versions.tf | 12 +++++
deployment/terraform-ansible/deploy-pulsar.yaml | 4 +-
11 files changed, 89 insertions(+), 70 deletions(-)
diff --git a/deployment/terraform-ansible/aws/ansible.cfg b/deployment/terraform-ansible/aws/ansible.cfg
index 8807c29..0301bf7 100644
--- a/deployment/terraform-ansible/aws/ansible.cfg
+++ b/deployment/terraform-ansible/aws/ansible.cfg
@@ -18,7 +18,7 @@
#
[defaults]
-private_key_file=~/.ssh/id_rsa
+private_key_file=~/.ssh/pulsar_aws
host_key_checking=false
[privilege_escalation]
diff --git a/deployment/terraform-ansible/aws/instances.tf b/deployment/terraform-ansible/aws/instances.tf
index bc42a0b..125cc23 100644
--- a/deployment/terraform-ansible/aws/instances.tf
+++ b/deployment/terraform-ansible/aws/instances.tf
@@ -18,53 +18,53 @@
#
resource "aws_instance" "zookeeper" {
- ami = "${var.aws_ami}"
- instance_type = "${var.instance_types["zookeeper"]}"
- key_name = "${aws_key_pair.default.id}"
- subnet_id = "${aws_subnet.default.id}"
- vpc_security_group_ids = ["${aws_security_group.default.id}"]
- count = "${var.num_zookeeper_nodes}"
+ ami = var.aws_ami
+ instance_type = var.instance_types["zookeeper"]
+ key_name = aws_key_pair.default.id
+ subnet_id = aws_subnet.default.id
+ vpc_security_group_ids = [aws_security_group.default.id]
+ count = var.num_zookeeper_nodes
- tags {
+ tags = {
Name = "zookeeper-${count.index + 1}"
}
}
resource "aws_instance" "bookie" {
- ami = "${var.aws_ami}"
- instance_type = "${var.instance_types["bookie"]}"
- key_name = "${aws_key_pair.default.id}"
- subnet_id = "${aws_subnet.default.id}"
- vpc_security_group_ids = ["${aws_security_group.default.id}"]
- count = "${var.num_bookie_nodes}"
+ ami = var.aws_ami
+ instance_type = var.instance_types["bookie"]
+ key_name = aws_key_pair.default.id
+ subnet_id = aws_subnet.default.id
+ vpc_security_group_ids = [aws_security_group.default.id]
+ count = var.num_bookie_nodes
- tags {
+ tags = {
Name = "bookie-${count.index + 1}"
}
}
resource "aws_instance" "broker" {
- ami = "${var.aws_ami}"
- instance_type = "${var.instance_types["broker"]}"
- key_name = "${aws_key_pair.default.id}"
- subnet_id = "${aws_subnet.default.id}"
- vpc_security_group_ids = ["${aws_security_group.default.id}"]
- count = "${var.num_broker_nodes}"
+ ami = var.aws_ami
+ instance_type = var.instance_types["broker"]
+ key_name = aws_key_pair.default.id
+ subnet_id = aws_subnet.default.id
+ vpc_security_group_ids = [aws_security_group.default.id]
+ count = var.num_broker_nodes
- tags {
+ tags = {
Name = "broker-${count.index + 1}"
}
}
resource "aws_instance" "proxy" {
- ami = "${var.aws_ami}"
- instance_type = "${var.instance_types["proxy"]}"
- key_name = "${aws_key_pair.default.id}"
- subnet_id = "${aws_subnet.default.id}"
- vpc_security_group_ids = ["${aws_security_group.default.id}"]
- count = "${var.num_proxy_nodes}"
+ ami = var.aws_ami
+ instance_type = var.instance_types["proxy"]
+ key_name = aws_key_pair.default.id
+ subnet_id = aws_subnet.default.id
+ vpc_security_group_ids = [aws_security_group.default.id]
+ count = var.num_proxy_nodes
- tags {
+ tags = {
Name = "proxy-${count.index + 1}"
}
}
diff --git a/deployment/terraform-ansible/aws/keys.tf b/deployment/terraform-ansible/aws/keys.tf
index 02263a4..d81a2da 100644
--- a/deployment/terraform-ansible/aws/keys.tf
+++ b/deployment/terraform-ansible/aws/keys.tf
@@ -23,6 +23,6 @@ resource "random_id" "key_pair_name" {
}
resource "aws_key_pair" "default" {
- key_name = "${random_id.key_pair_name.hex}"
- public_key = "${file(var.public_key_path)}"
+ key_name = random_id.key_pair_name.hex
+ public_key = file(var.public_key_path)
}
diff --git a/deployment/terraform-ansible/aws/network.tf b/deployment/terraform-ansible/aws/network.tf
index 97c6624..75d91cf 100644
--- a/deployment/terraform-ansible/aws/network.tf
+++ b/deployment/terraform-ansible/aws/network.tf
@@ -18,82 +18,82 @@
#
resource "aws_vpc" "pulsar_vpc" {
- cidr_block = "${var.base_cidr_block}"
+ cidr_block = var.base_cidr_block
enable_dns_support = true
enable_dns_hostnames = true
- tags {
+ tags = {
Name = "Pulsar-VPC"
}
}
resource "aws_subnet" "default" {
- vpc_id = "${aws_vpc.pulsar_vpc.id}"
- cidr_block = "${cidrsubnet(var.base_cidr_block, 8, 2)}"
- availability_zone = "${var.availability_zone}"
+ vpc_id = aws_vpc.pulsar_vpc.id
+ cidr_block = cidrsubnet(var.base_cidr_block, 8, 2)
+ availability_zone = var.availability_zone
map_public_ip_on_launch = true
- tags {
+ tags = {
Name = "Pulsar-Subnet"
}
}
resource "aws_route_table" "default" {
- vpc_id = "${aws_vpc.pulsar_vpc.id}"
+ vpc_id = aws_vpc.pulsar_vpc.id
- tags {
+ tags = {
Name = "Pulsar-Route-Table"
}
}
resource "aws_route" "default" {
- route_table_id = "${aws_route_table.default.id}"
+ route_table_id = aws_route_table.default.id
destination_cidr_block = "0.0.0.0/0"
- nat_gateway_id = "${aws_nat_gateway.default.id}"
+ nat_gateway_id = aws_nat_gateway.default.id
}
resource "aws_route_table_association" "default" {
- subnet_id = "${aws_subnet.default.id}"
- route_table_id = "${aws_vpc.pulsar_vpc.main_route_table_id}"
+ subnet_id = aws_subnet.default.id
+ route_table_id = aws_vpc.pulsar_vpc.main_route_table_id
}
/* Misc */
resource "aws_eip" "default" {
vpc = true
- depends_on = ["aws_internet_gateway.default"]
+ depends_on = [aws_internet_gateway.default]
}
resource "aws_internet_gateway" "default" {
- vpc_id = "${aws_vpc.pulsar_vpc.id}"
+ vpc_id = aws_vpc.pulsar_vpc.id
- tags {
+ tags = {
Name = "Pulsar-Internet-Gateway"
}
}
resource "aws_nat_gateway" "default" {
- allocation_id = "${aws_eip.default.id}"
- subnet_id = "${aws_subnet.default.id}"
- depends_on = ["aws_internet_gateway.default"]
+ allocation_id = aws_eip.default.id
+ subnet_id = aws_subnet.default.id
+ depends_on = [aws_internet_gateway.default]
- tags {
+ tags = {
Name = "Pulsar-NAT-Gateway"
}
}
/* Public internet route */
resource "aws_route" "internet_access" {
- route_table_id = "${aws_vpc.pulsar_vpc.main_route_table_id}"
+ route_table_id = aws_vpc.pulsar_vpc.main_route_table_id
destination_cidr_block = "0.0.0.0/0"
- gateway_id = "${aws_internet_gateway.default.id}"
+ gateway_id = aws_internet_gateway.default.id
}
/* Load balancer */
resource "aws_elb" "default" {
name = "pulsar-elb"
- instances = ["${aws_instance.proxy.*.id}"]
- security_groups = ["${aws_security_group.elb.id}"]
- subnets = ["${aws_subnet.default.id}"]
+ instances = aws_instance.proxy[*].id
+ security_groups = [aws_security_group.elb.id]
+ subnets = [aws_subnet.default.id]
listener {
instance_port = 8080
@@ -111,7 +111,7 @@ resource "aws_elb" "default" {
cross_zone_load_balancing = false
- tags {
+ tags = {
Name = "Pulsar-Load-Balancer"
}
}
diff --git a/deployment/terraform-ansible/aws/provider.tf b/deployment/terraform-ansible/aws/provider.tf
index 3c01acc..bf1357a 100644
--- a/deployment/terraform-ansible/aws/provider.tf
+++ b/deployment/terraform-ansible/aws/provider.tf
@@ -18,6 +18,5 @@
#
provider "aws" {
- region = "${var.region}"
- version = "1.5"
+ region = var.region
}
diff --git a/deployment/terraform-ansible/aws/security.tf b/deployment/terraform-ansible/aws/security.tf
index 1f1e261..89c11b1 100644
--- a/deployment/terraform-ansible/aws/security.tf
+++ b/deployment/terraform-ansible/aws/security.tf
@@ -19,7 +19,7 @@
resource "aws_security_group" "elb" {
name = "pulsar-elb"
- vpc_id = "${aws_vpc.pulsar_vpc.id}"
+ vpc_id = aws_vpc.pulsar_vpc.id
ingress {
from_port = 6650
@@ -45,7 +45,7 @@ resource "aws_security_group" "elb" {
resource "aws_security_group" "default" {
name = "pulsar-terraform"
- vpc_id = "${aws_vpc.pulsar_vpc.id}"
+ vpc_id = aws_vpc.pulsar_vpc.id
# SSH access from anywhere
ingress {
@@ -60,7 +60,7 @@ resource "aws_security_group" "default" {
from_port = 0
to_port = 65535
protocol = "tcp"
- cidr_blocks = ["${var.base_cidr_block}"]
+ cidr_blocks = [var.base_cidr_block]
}
# outbound internet access
@@ -71,7 +71,7 @@ resource "aws_security_group" "default" {
cidr_blocks = ["0.0.0.0/0"]
}
- tags {
+ tags = {
Name = "Pulsar-Security-Group"
}
}
diff --git a/deployment/terraform-ansible/aws/setup-disk.yaml b/deployment/terraform-ansible/aws/setup-disk.yaml
index 9c6ebb9..34342a4 100644
--- a/deployment/terraform-ansible/aws/setup-disk.yaml
+++ b/deployment/terraform-ansible/aws/setup-disk.yaml
@@ -22,9 +22,16 @@
connection: ssh
become: true
tasks:
- - command: >
- tuned-adm profile latency-performance
- - name: Create and mount disks
+ - name: Set performance profile
+ command: tuned-adm profile latency-performance
+ - name: Format disks
+ filesystem:
+ fstype: xfs
+ dev: '{{ item }}'
+ with_items:
+ - '/dev/nvme0n1'
+ - '/dev/nvme1n1'
+ - name: Mount disks
mount:
path: "{{ item.path }}"
src: "{{ item.src }}"
@@ -34,3 +41,4 @@
with_items:
- { path: "/mnt/journal", src: "/dev/nvme0n1" }
- { path: "/mnt/storage", src: "/dev/nvme1n1" }
+
diff --git a/deployment/terraform-ansible/aws/terraform.tfvars b/deployment/terraform-ansible/aws/terraform.tfvars
index 18325f1..49aed5c 100644
--- a/deployment/terraform-ansible/aws/terraform.tfvars
+++ b/deployment/terraform-ansible/aws/terraform.tfvars
@@ -17,7 +17,7 @@
# under the License.
#
-public_key_path = "~/.ssh/id_rsa.pub"
+public_key_path = "~/.ssh/pulsar_aws.pub"
region = "us-west-2"
availability_zone = "us-west-2a"
aws_ami = "ami-9fa343e7"
@@ -29,7 +29,7 @@ base_cidr_block = "10.0.0.0/16"
instance_types = {
"zookeeper" = "t3.small"
- "bookie" = "i3.xlarge"
+ "bookie" = "i3.4xlarge"
"broker" = "c5.2xlarge"
"proxy" = "c5.2xlarge"
}
diff --git a/deployment/terraform-ansible/aws/variables.tf b/deployment/terraform-ansible/aws/variables.tf
index ec98b69..ef2e73f 100644
--- a/deployment/terraform-ansible/aws/variables.tf
+++ b/deployment/terraform-ansible/aws/variables.tf
@@ -62,7 +62,7 @@ variable "num_proxy_nodes" {
}
variable "instance_types" {
- type = "map"
+ type = map(string)
}
variable "base_cidr_block" {
diff --git a/deployment/terraform-ansible/aws/versions.tf b/deployment/terraform-ansible/aws/versions.tf
new file mode 100644
index 0000000..8ce589b
--- /dev/null
+++ b/deployment/terraform-ansible/aws/versions.tf
@@ -0,0 +1,12 @@
+terraform {
+ required_providers {
+ aws = {
+ source = "hashicorp/aws"
+ version = ">=3.3.0"
+ }
+ random = {
+ source = "hashicorp/random"
+ }
+ }
+ required_version = ">= 0.13"
+}
diff --git a/deployment/terraform-ansible/deploy-pulsar.yaml b/deployment/terraform-ansible/deploy-pulsar.yaml
index c016d54..ae1243a 100644
--- a/deployment/terraform-ansible/deploy-pulsar.yaml
+++ b/deployment/terraform-ansible/deploy-pulsar.yaml
@@ -42,7 +42,7 @@
pulsar_version: "2.8.1"
- name: Download Pulsar binary package
unarchive:
- src: https://www.apache.org/dyn/mirrors/mirrors.cgi?action=download&filename=pulsar/pulsar-{{ pulsar_version }}/apache-pulsar-{{ pulsar_version }}-bin.tar.gz
+ src: https://archive.apache.org/dist/pulsar/pulsar-{{ pulsar_version }}/apache-pulsar-{{ pulsar_version }}-bin.tar.gz
remote_src: yes
dest: /opt/pulsar
extra_opts: ["--strip-components=1"]
@@ -132,7 +132,7 @@
- connectors
- name: Download Pulsar IO packages
get_url:
- url: https://www.apache.org/dyn/mirrors/mirrors.cgi?action=download&filename=pulsar/pulsar-{{ pulsar_version }}/connectors/pulsar-io-{{ item }}-{{ pulsar_version }}.nar
+ url: https://archive.apache.org/dist/pulsar/pulsar-{{ pulsar_version }}/connectors/pulsar-io-{{ item }}-{{ pulsar_version }}.nar
dest: /opt/pulsar/connectors/pulsar-io-{{ item }}-{{ pulsar_version }}.nar
loop:
# - aerospike