You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by GitBox <gi...@apache.org> on 2021/04/10 18:59:34 UTC
[GitHub] [airflow-ci-infra] hmike96 opened a new pull request #14: Automation for runner AMI creation and CI Infra
hmike96 opened a new pull request #14:
URL: https://github.com/apache/airflow-ci-infra/pull/14
Packer for AMI creation and Terraform for CI Infra
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [airflow-ci-infra] hmike96 commented on pull request #14: Automation for runner AMI creation and CI Infra
Posted by GitBox <gi...@apache.org>.
hmike96 commented on pull request #14:
URL: https://github.com/apache/airflow-ci-infra/pull/14#issuecomment-829352034
Closing this as packer is already merged will open a new one when terraform is finished for aws infrastructure.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [airflow-ci-infra] xinbinhuang commented on a change in pull request #14: Automation for runner AMI creation and CI Infra
Posted by GitBox <gi...@apache.org>.
xinbinhuang commented on a change in pull request #14:
URL: https://github.com/apache/airflow-ci-infra/pull/14#discussion_r621384944
##########
File path: github-runner-ami/packer/files/install-dependencies.sh
##########
@@ -17,6 +20,10 @@
# specific language governing permissions and limitations
# under the License.
+<<<<<<< HEAD
+sudo apt-get update
+sudo apt-get install \
+=======
Review comment:
Are these merge conflict lines?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [airflow-ci-infra] hmike96 closed pull request #14: Automation for runner AMI creation and CI Infra
Posted by GitBox <gi...@apache.org>.
hmike96 closed pull request #14:
URL: https://github.com/apache/airflow-ci-infra/pull/14
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [airflow-ci-infra] ashb commented on a change in pull request #14: Automation for runner AMI creation and CI Infra
Posted by GitBox <gi...@apache.org>.
ashb commented on a change in pull request #14:
URL: https://github.com/apache/airflow-ci-infra/pull/14#discussion_r611449392
##########
File path: github-runner-ami/packer/ubuntu2004.pkr.hcl
##########
@@ -0,0 +1,132 @@
+variable "vpc_id" {
+ type = string
+}
+variable "ami_name" {
+ type = string
+}
+variable "aws_region" {
+ type = string
+}
+variable "subnet_id" {
+ type = string
+}
+variable "packer_role_arn" {
+ type = string
+}
+variable "runner_version" {
+ type = string
+}
+variable "kms_key_arn" {
+ type = string
+}
+variable "session_manager_instance_profile_name" {
+ type = string
+}
+
+source "amazon-ebs" "runner_builder" {
+ assume_role {
+ role_arn = var.packer_role_arn
+ session_name = var.runner_version
+ }
+ #access_key = ""
+ #secret_key = ""
+ region = var.aws_region
+ ami_name = "${var.ami_name}-${var.runner_version}"
+ ami_regions = [var.aws_region]
+ tag {
+ key = "ami"
+ value = "github-runner-ami"
+ }
+ encrypt_boot = true
+ kms_key_id = var.kms_key_arn
+ instance_type = "t2.micro"
+ communicator = "ssh"
+ ssh_username = "ubuntu"
+ ssh_interface = "session_manager"
+ iam_instance_profile = var.session_manager_instance_profile_name
+ subnet_id = var.subnet_id
+ vpc_id = var.vpc_id
+ source_ami_filter {
+ filters = {
+ virtualization-type = "hvm"
+ name = "ubuntu/images/*buntu-focal-20.04-amd64-server-*"
+ root-device-type = "ebs"
+ }
+ owners = ["099720109477"]
+ most_recent = true
+ }
+}
+
+build {
+ sources = [
+ "source.amazon-ebs.runner_builder"
+ ]
+
+ provisioner "shell" {
+ inline = [
+ "echo Connected via SSM at '${build.User}@${build.Host}:${build.Port}'"
+ ]
+ }
+ provisioner "file" {
+ destination = "/usr/local/sbin/mounts_setup.sh"
+ source = "./files/mounts_setup.sh"
+ }
+ provisioner "shell" {
+ inline = ["sh mounts_setup.sh"]
+ }
+ provisioner "file" {
+ destination = "/etc/systemd/system/actions.runner.service"
+ source = "./files/actions.runner.service"
+ }
+ provisioner "file" {
+ destination = "/usr/local/sbin/runner-cleanup-workdir.sh"
+ source = "./files/runner-cleanup-workdir.sh"
+ }
+ provisioner "file" {
+ destination = "/usr/local/sbin/stop-runner-if-no-job.sh"
+ source = "./files/stop-runner-if-no-job.sh"
+ }
+ provisioner "file" {
+ destination = "/etc/sudoers.d/runner"
+ source = "./files/runner"
+ }
+ provisioner "file" {
+ destination = "/etc/iptables/rules.v4"
+ source = "./files/rules.v4"
+ }
+ provisioner "file" {
+ destination = "/usr/local/sbin/actions-runner-ec2-reporting.sh"
+ source = "./files/actions-runner-ec2-reporting.sh"
+ }
+ provisioner "file" {
+ destination = "/etc/cron.d/cloudwatch-metrics-github-runners"
+ source = "./files/cloudwatch-metrics-github-runners"
+ }
+ provisioner "file" {
+ destination = "/etc/systemd/system/actions.runner-supervisor.service"
+ source = "./files/actions.runner-supervisor.service"
+ }
+ provisioner "file" {
+ destination = "/usr/local/sbin/set-file-permissions.sh"
+ source = "./files/set-file-permissions.sh"
+ }
+ provisioner "file" {
+ destination = "/usr/local/sbin/timber.key"
+ source = "./files/timber.key"
+ }
+ provisioner "file" {
+ destination = "/usr/local/sbin/source-list-additions.sh"
+ source = "./files/source-list-additions.sh"
+ }
+ provisioner "file" {
+ destination = "/usr/local/sbin/install-dependencies.sh"
+ source = "./files/install-dependencies.sh"
+ }
+ provisioner "file" {
+ destination = "/usr/local/sbin/runner_bootstrap.sh"
+ source = "./files/runner_bootstrap.sh"
+ }
+ provisioner "shell-local" {
Review comment:
```suggestion
provisioner "shell" {
```
shell-local runs on the local machine -- I don't think that is what you want here :)
##########
File path: github-runner-ami/packer/files/runner_bootstrap.sh
##########
@@ -0,0 +1,37 @@
+
+URL=$(curl -s https://api.github.com/repos/docker/compose/releases/latest | jq -r '.assets[].browser_download_url | select(endswith("docker-compose-Linux-x86_64"))')
+curl -L $URL -o /usr/local/bin/docker-compose
+chmod +x /usr/local/bin/docker-compose
+set -exu -o pipefail
+echo "AWS_DEFAULT_REGION=$(cloud-init query region)" >> /etc/environment
+# Set an env var (that is visible in runners) that will let us know we are on a self-hosted runner
+echo 'AIRFLOW_SELF_HOSTED_RUNNER="[\"self-hosted\"]"' >> /etc/environment
+set -a
+. /etc/environment
+set +a
+systemctl daemon-reload
+set -exu -o pipefail
+usermod -G docker -a runner
+mkdir -p ~runner/actions-runner
+find ~runner -exec chown runner: {} +
+cd ~runner/actions-runner
+RUNNER_VERSION="$0"
+curl -L "https://github.com/ashb/runner/releases/download/v${RUNNER_VERSION}/actions-runner-linux-x64-${RUNNER_VERSION}.tar.gz" | tar -zx
+set -a
+. /etc/environment
+set +a
+aws s3 cp s3://airflow-ci-assets/runner-supervisor.py /opt/runner-supervisor/bin/runner-supervisor
+chmod 755 /opt/runner-supervisor/bin/runner-supervisor
+# Log in to a paid docker user to get unlimited docker pulls
+aws ssm get-parameter --with-decryption --name /runners/apache/airflow/dockerPassword | \
+jq .Parameter.Value -r | \
+sudo -u runner docker login --username airflowcirunners --password-stdin
+2.277.1-airflow1
+systemctl enable --now iptables.service
+# Restart docker after applying the user firewall -- else some rules/chains might be list!
+systemctl restart docker.service
+systemctl enable now vector.service
+systemctl enable --now actions.runner.service
+echo "Pre-loading commonly used docker images from S3"
+set -eux -o pipefail
+aws s3 cp s3://airflow-ci-assets/pre-baked-images.tar.gz - | docker load
Review comment:
```suggestion
```
(We got rid of this)
##########
File path: github-runner-ami/packer/ubuntu2004.pkr.hcl
##########
@@ -0,0 +1,132 @@
+variable "vpc_id" {
+ type = string
+}
+variable "ami_name" {
+ type = string
+}
+variable "aws_region" {
+ type = string
+}
+variable "subnet_id" {
+ type = string
+}
+variable "packer_role_arn" {
+ type = string
+}
+variable "runner_version" {
+ type = string
+}
+variable "kms_key_arn" {
+ type = string
+}
+variable "session_manager_instance_profile_name" {
+ type = string
+}
+
+source "amazon-ebs" "runner_builder" {
+ assume_role {
+ role_arn = var.packer_role_arn
+ session_name = var.runner_version
+ }
+ #access_key = ""
+ #secret_key = ""
+ region = var.aws_region
+ ami_name = "${var.ami_name}-${var.runner_version}"
+ ami_regions = [var.aws_region]
+ tag {
+ key = "ami"
+ value = "github-runner-ami"
+ }
+ encrypt_boot = true
+ kms_key_id = var.kms_key_arn
+ instance_type = "t2.micro"
+ communicator = "ssh"
+ ssh_username = "ubuntu"
+ ssh_interface = "session_manager"
+ iam_instance_profile = var.session_manager_instance_profile_name
+ subnet_id = var.subnet_id
+ vpc_id = var.vpc_id
+ source_ami_filter {
+ filters = {
+ virtualization-type = "hvm"
+ name = "ubuntu/images/*buntu-focal-20.04-amd64-server-*"
+ root-device-type = "ebs"
+ }
+ owners = ["099720109477"]
+ most_recent = true
+ }
+}
+
+build {
+ sources = [
+ "source.amazon-ebs.runner_builder"
+ ]
+
+ provisioner "shell" {
+ inline = [
+ "echo Connected via SSM at '${build.User}@${build.Host}:${build.Port}'"
+ ]
+ }
+ provisioner "file" {
+ destination = "/usr/local/sbin/mounts_setup.sh"
+ source = "./files/mounts_setup.sh"
+ }
+ provisioner "shell" {
+ inline = ["sh mounts_setup.sh"]
+ }
Review comment:
```suggestion
```
These mount suggestions need to be done at "runtime", not image build time, so these need to stay in the user data script
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org