You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by ma...@apache.org on 2019/08/15 22:21:09 UTC
[incubator-mxnet-ci] 01/01: Add unix slave creation
This is an automated email from the ASF dual-hosted git repository.
marcoabreu pushed a commit to branch unix-slave-creation
in repository https://gitbox.apache.org/repos/asf/incubator-mxnet-ci.git
commit 628483f16f24f0d07d2172b4f4849ef524c8e4a9
Author: Marco de Abreu <ma...@gmail.com>
AuthorDate: Fri Aug 16 00:17:18 2019 +0200
Add unix slave creation
---
tools/jenkins-slave-creation-unix/README.md | 18 ++
.../conf-ubuntu-cpu-c5/infrastructure.tfvars | 34 ++
.../infrastructure_backend.tfvars | 22 ++
.../conf-ubuntu-cpu-c5/install.sh | 128 ++++++++
.../conf-ubuntu-cpu-c5/shell-variables.sh | 22 ++
.../conf-ubuntu-gpu-g3/infrastructure.tfvars | 34 ++
.../infrastructure_backend.tfvars | 22 ++
.../conf-ubuntu-gpu-g3/install.sh | 144 +++++++++
.../conf-ubuntu-gpu-g3/shell-variables.sh | 22 ++
.../conf-ubuntu-gpu-p3/infrastructure.tfvars | 34 ++
.../infrastructure_backend.tfvars | 22 ++
.../conf-ubuntu-gpu-p3/install.sh | 141 +++++++++
.../conf-ubuntu-gpu-p3/shell-variables.sh | 22 ++
tools/jenkins-slave-creation-unix/create_slave.sh | 26 ++
.../jenkins-slave-creation-unix/infrastructure.tf | 326 ++++++++++++++++++++
tools/jenkins-slave-creation-unix/init.sh | 23 ++
.../scripts/create_slave.py | 106 +++++++
.../scripts/deploy/launch-autoconnect.sh | 22 ++
.../scripts/deploy/launch-ccache-mount-efs.sh | 40 +++
.../scripts/deploy/slave-autoconnect.py | 342 +++++++++++++++++++++
20 files changed, 1550 insertions(+)
diff --git a/tools/jenkins-slave-creation-unix/README.md b/tools/jenkins-slave-creation-unix/README.md
new file mode 100644
index 0000000..6b11f58
--- /dev/null
+++ b/tools/jenkins-slave-creation-unix/README.md
@@ -0,0 +1,18 @@
+<!--- Licensed to the Apache Software Foundation (ASF) under one -->
+<!--- or more contributor license agreements. See the NOTICE file -->
+<!--- distributed with this work for additional information -->
+<!--- regarding copyright ownership. The ASF licenses this file -->
+<!--- to you under the Apache License, Version 2.0 (the -->
+<!--- "License"); you may not use this file except in compliance -->
+<!--- with the License. You may obtain a copy of the License at -->
+
+<!--- http://www.apache.org/licenses/LICENSE-2.0 -->
+
+<!--- Unless required by applicable law or agreed to in writing, -->
+<!--- software distributed under the License is distributed on an -->
+<!--- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->
+<!--- KIND, either express or implied. See the License for the -->
+<!--- specific language governing permissions and limitations -->
+<!--- under the License. -->
+
+This Terraform setup will spawn an instance that is ready to be saved into an AMI to create a Jenkins slave.
\ No newline at end of file
diff --git a/tools/jenkins-slave-creation-unix/conf-ubuntu-cpu-c5/infrastructure.tfvars b/tools/jenkins-slave-creation-unix/conf-ubuntu-cpu-c5/infrastructure.tfvars
new file mode 100644
index 0000000..0aea681
--- /dev/null
+++ b/tools/jenkins-slave-creation-unix/conf-ubuntu-cpu-c5/infrastructure.tfvars
@@ -0,0 +1,34 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+key_name = "REDACTED"
+key_path = "~/.ssh/REDACTED"
+instance_type = "c5.18xlarge"
+
+additional_security_group_ids = [
+ "sg-5d83d421", # VPC default
+ "sg-REDACTED" # REDACTED
+]
+
+s3_config_bucket = "mxnet-ci-slave-dev"
+s3_config_filename = "ubuntu-cpu-c5-config.tar.bz2"
+slave_install_script = "conf-ubuntu-cpu-c5/install.sh"
+shell_variables_file = "conf-ubuntu-cpu-c5/shell-variables.sh"
+ami = "ami-bd8f33c5" # ftp://64.50.236.216/pub/ubuntu-cloud-images/query/xenial/server/released.txt
+instance_name = "Slave-base_Ubuntu-CPU-C5"
+aws_region = "us-west-2"
+secret_manager_docker_hub_arn = "arn:aws:secretsmanager:us-west-2:REDACTED:secret:REDACTED"
diff --git a/tools/jenkins-slave-creation-unix/conf-ubuntu-cpu-c5/infrastructure_backend.tfvars b/tools/jenkins-slave-creation-unix/conf-ubuntu-cpu-c5/infrastructure_backend.tfvars
new file mode 100644
index 0000000..3e07798
--- /dev/null
+++ b/tools/jenkins-slave-creation-unix/conf-ubuntu-cpu-c5/infrastructure_backend.tfvars
@@ -0,0 +1,22 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Terraform backend configuration. See See https://www.terraform.io/docs/backends/config.html for
+# more details. Reason being that config can not be interpolated during backend init
+
+bucket = "mxnet-ci-slave-dev-tfstate"
+region = "us-west-2"
diff --git a/tools/jenkins-slave-creation-unix/conf-ubuntu-cpu-c5/install.sh b/tools/jenkins-slave-creation-unix/conf-ubuntu-cpu-c5/install.sh
new file mode 100644
index 0000000..185b5bf
--- /dev/null
+++ b/tools/jenkins-slave-creation-unix/conf-ubuntu-cpu-c5/install.sh
@@ -0,0 +1,128 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+set -x
+
+# Load variables
+source /var/lib/cloud/instance/scripts/part-001
+
+whoami
+
+#Setup jenkins user
+sudo useradd jenkins_slave
+
+#Prevent log in
+sudo usermod -L jenkins_slave
+sudo mkdir -p /home/jenkins_slave/remoting
+sudo chown -R jenkins_slave:jenkins_slave /home/jenkins_slave
+
+#Remove preinstalled packaged
+sudo apt-get -y purge openjdk*
+sudo apt-get -y purge nvidia*
+echo "Purged packages"
+
+#Add third party repositories
+sudo add-apt-repository -y ppa:graphics-drivers/ppa
+sudo curl -fsSL https://apt.dockerproject.org/gpg | sudo apt-key add -
+sudo apt-key fingerprint 58118E89F3A912897C070ADBF76221572C52609D
+sudo add-apt-repository "deb https://apt.dockerproject.org/repo/ ubuntu-xenial main"
+sudo apt-get update
+echo "Added third party repositories"
+
+#Install htop
+sudo apt-get -y install htop
+
+#Install java
+sudo apt-get -y install openjdk-8-jre
+
+#Install git
+sudo apt-get -y install git
+sudo -H -S -u jenkins_slave git config --global user.email "mxnet-ci"
+sudo -H -S -u jenkins_slave git config --global user.name "mxnet-ci"
+
+#Install python3, pip3 and dependencies for auto-connect.py
+sudo apt-get -y install python3 python3-pip
+sudo pip3 install boto3 python-jenkins joblib docker
+
+echo "Installed htop, java, git and python"
+
+
+#Install docker engine
+sudo apt-get install -y apt-transport-https ca-certificates curl software-properties-common
+curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+sudo apt-key fingerprint 0EBFCD88
+sudo add-apt-repository \
+ "deb [arch=amd64] https://download.docker.com/linux/ubuntu \
+ $(lsb_release -cs) \
+ stable"
+sudo apt-get update
+sudo apt-get install -y docker-ce
+sudo usermod -aG docker jenkins_slave
+sudo systemctl enable docker #Enable docker to start on startup
+sudo service docker restart
+echo "Installed docker engine"
+
+# Download additional scripts
+sudo apt-get -y install awscli
+tmpdir=$(mktemp -d)
+aws s3 cp --quiet s3://$S3_CONFIG_BUCKET/$S3_CONFIG_FILE $tmpdir/scripts.tar.bz2
+
+mkdir /home/jenkins_slave/scripts
+tar -C /home/jenkins_slave/scripts -xjf $tmpdir/scripts.tar.bz2
+find /home/jenkins_slave/scripts -type f -exec chmod 744 {} \;
+chown -R jenkins_slave.jenkins_slave /home/jenkins_slave
+
+# Set up swap of 100GB
+fallocate -l 100G /swapfile
+chown root:root /swapfile
+chmod 0600 /swapfile
+mkswap /swapfile
+swapon /swapfile
+echo /swapfile none swap sw 0 0 >> /etc/fstab
+
+# Add auto-connecting to slave to startup
+touch /home/jenkins_slave/auto-connect.log
+chown -R jenkins_slave:jenkins_slave /home/jenkins_slave/
+echo "@reboot jenkins_slave /home/jenkins_slave/scripts/launch-autoconnect.sh" > /etc/cron.d/jenkins-start-slave
+
+# Install NFS client for EFS
+sudo apt-get -y install nfs-common
+
+# Auto-mount EFS on startup
+echo "@reboot root /home/jenkins_slave/scripts/launch-ccache-mount-efs.sh" > /etc/cron.d/ccache-mount-efs
+
+# Write instructions to home dir
+readme="Please use the following command in your cloud-init-script to specify the jenkins master address:
+#!/bin/bash
+echo 'http://jenkins.mxnet-ci.amazon-ml.com/' > /home/jenkins_slave/jenkins_master_url
+echo 'http://jenkins-priv.mxnet-ci.amazon-ml.com/' > /home/jenkins_slave/jenkins_master_private_url
+
+
+Optional:
+echo 'mxnet-linux-cpu10' > /home/jenkins_slave/jenkins_slave_name
+"
+echo "$readme" > /home/ubuntu/readme.txt
+
+
+echo "Setup completed"
+
+# For testing use reboot, but lateron just turn off the instance to prepare AMI generation
+# reboot
+shutdown -h now
diff --git a/tools/jenkins-slave-creation-unix/conf-ubuntu-cpu-c5/shell-variables.sh b/tools/jenkins-slave-creation-unix/conf-ubuntu-cpu-c5/shell-variables.sh
new file mode 100644
index 0000000..916138c
--- /dev/null
+++ b/tools/jenkins-slave-creation-unix/conf-ubuntu-cpu-c5/shell-variables.sh
@@ -0,0 +1,22 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+
+export S3_CONFIG_BUCKET="mxnet-ci-slave-dev"
+export S3_CONFIG_FILE="ubuntu-cpu-c5-config.tar.bz2"
diff --git a/tools/jenkins-slave-creation-unix/conf-ubuntu-gpu-g3/infrastructure.tfvars b/tools/jenkins-slave-creation-unix/conf-ubuntu-gpu-g3/infrastructure.tfvars
new file mode 100644
index 0000000..5c8fe54
--- /dev/null
+++ b/tools/jenkins-slave-creation-unix/conf-ubuntu-gpu-g3/infrastructure.tfvars
@@ -0,0 +1,34 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+key_name = "REDACTED"
+key_path = "~/.ssh/REDACTED"
+instance_type = "g3.8xlarge"
+
+additional_security_group_ids = [
+ "sg-5d83d421", # VPC default
+ "REDACTED" # REDACTED
+]
+
+s3_config_bucket = "mxnet-ci-slave-dev"
+s3_config_filename = "ubuntu-gpu-g3-config.tar.bz2"
+slave_install_script = "conf-ubuntu-gpu-g3/install.sh"
+shell_variables_file = "conf-ubuntu-gpu-g3/shell-variables.sh"
+ami = "ami-bd8f33c5" # ftp://64.50.236.216/pub/ubuntu-cloud-images/query/xenial/server/released.txt
+instance_name = "Slave-base_Ubuntu-GPU-G3"
+aws_region = "us-west-2"
+secret_manager_docker_hub_arn = "arn:aws:secretsmanager:us-west-2:REDACTED:secret:REDACTED"
diff --git a/tools/jenkins-slave-creation-unix/conf-ubuntu-gpu-g3/infrastructure_backend.tfvars b/tools/jenkins-slave-creation-unix/conf-ubuntu-gpu-g3/infrastructure_backend.tfvars
new file mode 100644
index 0000000..3e07798
--- /dev/null
+++ b/tools/jenkins-slave-creation-unix/conf-ubuntu-gpu-g3/infrastructure_backend.tfvars
@@ -0,0 +1,22 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Terraform backend configuration. See See https://www.terraform.io/docs/backends/config.html for
+# more details. Reason being that config can not be interpolated during backend init
+
+bucket = "mxnet-ci-slave-dev-tfstate"
+region = "us-west-2"
diff --git a/tools/jenkins-slave-creation-unix/conf-ubuntu-gpu-g3/install.sh b/tools/jenkins-slave-creation-unix/conf-ubuntu-gpu-g3/install.sh
new file mode 100644
index 0000000..6e60a7b
--- /dev/null
+++ b/tools/jenkins-slave-creation-unix/conf-ubuntu-gpu-g3/install.sh
@@ -0,0 +1,144 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+set -x
+
+# Load variables
+source /var/lib/cloud/instance/scripts/part-001
+
+whoami
+
+#Setup jenkins user
+sudo useradd jenkins_slave
+
+#Prevent log in
+sudo usermod -L jenkins_slave
+sudo mkdir -p /home/jenkins_slave/remoting
+sudo chown -R jenkins_slave:jenkins_slave /home/jenkins_slave
+
+#Remove preinstalled packaged
+sudo apt-get -y purge openjdk*
+sudo apt-get -y purge nvidia*
+echo "Purged packages"
+
+#Add third party repositories
+sudo add-apt-repository -y ppa:graphics-drivers/ppa
+sudo curl -fsSL https://apt.dockerproject.org/gpg | sudo apt-key add -
+sudo apt-key fingerprint 58118E89F3A912897C070ADBF76221572C52609D
+sudo add-apt-repository "deb https://apt.dockerproject.org/repo/ ubuntu-xenial main"
+sudo apt-get update
+echo "Added third party repositories"
+
+#Install htop
+sudo apt-get -y install htop
+
+#Install java
+sudo apt-get -y install openjdk-8-jre
+
+#Install git
+sudo apt-get -y install git
+sudo -H -S -u jenkins_slave git config --global user.email "mxnet-ci"
+sudo -H -S -u jenkins_slave git config --global user.name "mxnet-ci"
+
+#Install python3, pip3 and dependencies for auto-connect.py
+sudo apt-get -y install python3 python3-pip
+sudo pip3 install boto3 python-jenkins joblib docker
+
+echo "Installed htop, java, git and python"
+
+#Install nvidia drivers
+sudo apt-get -y install nvidia-410
+
+# TODO: - Disabled nvidia updates @ /etc/apt/apt.conf.d/50unattended-upgrades
+#Unattended-Upgrade::Package-Blacklist {
+#"nvidia-384";
+#"nvidia-opencl-icd-384";
+
+echo "Installed nvidia drivers"
+
+#Install docker engine
+sudo apt-get install -y apt-transport-https ca-certificates curl software-properties-common
+curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+sudo apt-key fingerprint 0EBFCD88
+sudo add-apt-repository \
+ "deb [arch=amd64] https://download.docker.com/linux/ubuntu \
+ $(lsb_release -cs) \
+ stable"
+sudo apt-get update
+sudo apt-get install -y docker-ce
+sudo usermod -aG docker jenkins_slave
+sudo systemctl enable docker #Enable docker to start on startup
+sudo service docker restart
+echo "Installed docker engine"
+
+# Add nvidia-docker and nvidia-docker-plugin
+curl -s -L https://nvidia.github.io/nvidia-docker/gpgkey | \
+ sudo apt-key add -
+distribution=$(. /etc/os-release;echo $ID$VERSION_ID)
+curl -s -L https://nvidia.github.io/nvidia-docker/$distribution/nvidia-docker.list | \
+ sudo tee /etc/apt/sources.list.d/nvidia-docker.list
+sudo apt-get update
+
+# Install nvidia-docker2 and reload the Docker daemon configuration
+sudo apt-get install -y nvidia-docker2
+sudo pkill -SIGHUP dockerd
+
+# Download additional scripts
+sudo apt-get -y install awscli
+tmpdir=$(mktemp -d)
+aws s3 cp --quiet s3://$S3_CONFIG_BUCKET/$S3_CONFIG_FILE $tmpdir/scripts.tar.bz2
+
+mkdir /home/jenkins_slave/scripts
+tar -C /home/jenkins_slave/scripts -xjf $tmpdir/scripts.tar.bz2
+find /home/jenkins_slave/scripts -type f -exec chmod 744 {} \;
+chown -R jenkins_slave.jenkins_slave /home/jenkins_slave
+
+# Set up swap of 50GB
+fallocate -l 50G /swapfile
+chown root:root /swapfile
+chmod 0600 /swapfile
+mkswap /swapfile
+swapon /swapfile
+echo /swapfile none swap sw 0 0 >> /etc/fstab
+
+# Add auto-connecting to slave to startup
+touch /home/jenkins_slave/auto-connect.log
+chown -R jenkins_slave:jenkins_slave /home/jenkins_slave/
+echo "@reboot jenkins_slave /home/jenkins_slave/scripts/launch-autoconnect.sh" > /etc/cron.d/jenkins-start-slave
+
+# Write instructions to home dir
+readme="Please use the following command in your cloud-init-script to specify the jenkins master
+address:
+#!/bin/bash
+echo 'http://jenkins.mxnet-ci.amazon-ml.com/' > /home/jenkins_slave/jenkins_master_url
+
+echo 'http://jenkins-priv.mxnet-ci.amazon-ml.com/' > /home/jenkins_slave/jenkins_master_private_url
+
+
+Optional:
+echo 'mxnet-linux-cpu10' > /home/jenkins_slave/jenkins_slave_name
+"
+echo "$readme" > /home/ubuntu/readme.txt
+
+echo "Setup completed"
+
+# For testing use reboot, but lateron just turn off the instance to prepare AMI generation
+# reboot
+shutdown -h now
diff --git a/tools/jenkins-slave-creation-unix/conf-ubuntu-gpu-g3/shell-variables.sh b/tools/jenkins-slave-creation-unix/conf-ubuntu-gpu-g3/shell-variables.sh
new file mode 100644
index 0000000..bda8818
--- /dev/null
+++ b/tools/jenkins-slave-creation-unix/conf-ubuntu-gpu-g3/shell-variables.sh
@@ -0,0 +1,22 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+
+export S3_CONFIG_BUCKET="mxnet-ci-slave-dev"
+export S3_CONFIG_FILE="ubuntu-gpu-g3-config.tar.bz2"
diff --git a/tools/jenkins-slave-creation-unix/conf-ubuntu-gpu-p3/infrastructure.tfvars b/tools/jenkins-slave-creation-unix/conf-ubuntu-gpu-p3/infrastructure.tfvars
new file mode 100644
index 0000000..6abc55b
--- /dev/null
+++ b/tools/jenkins-slave-creation-unix/conf-ubuntu-gpu-p3/infrastructure.tfvars
@@ -0,0 +1,34 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+key_name = "REDACTED"
+key_path = "~/.ssh/REDACTED"
+instance_type = "p3.2xlarge"
+
+additional_security_group_ids = [
+ "sg-5d83d421", # VPC default
+ "sg-REDACTED" # REDACTED
+]
+
+s3_config_bucket = "mxnet-ci-slave-dev"
+s3_config_filename = "ubuntu-gpu-p3-config.tar.bz2"
+slave_install_script = "conf-ubuntu-gpu-p3/install.sh"
+shell_variables_file = "conf-ubuntu-gpu-p3/shell-variables.sh"
+ami = "ami-bd8f33c5" # ftp://64.50.236.216/pub/ubuntu-cloud-images/query/xenial/server/released.txt
+instance_name = "Slave-base_Ubuntu-GPU-P3"
+aws_region = "us-west-2"
+secret_manager_docker_hub_arn = "arn:aws:secretsmanager:us-west-2:REDACTED:secret:REDACTED"
diff --git a/tools/jenkins-slave-creation-unix/conf-ubuntu-gpu-p3/infrastructure_backend.tfvars b/tools/jenkins-slave-creation-unix/conf-ubuntu-gpu-p3/infrastructure_backend.tfvars
new file mode 100644
index 0000000..3e07798
--- /dev/null
+++ b/tools/jenkins-slave-creation-unix/conf-ubuntu-gpu-p3/infrastructure_backend.tfvars
@@ -0,0 +1,22 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Terraform backend configuration. See See https://www.terraform.io/docs/backends/config.html for
+# more details. Reason being that config can not be interpolated during backend init
+
+bucket = "mxnet-ci-slave-dev-tfstate"
+region = "us-west-2"
diff --git a/tools/jenkins-slave-creation-unix/conf-ubuntu-gpu-p3/install.sh b/tools/jenkins-slave-creation-unix/conf-ubuntu-gpu-p3/install.sh
new file mode 100644
index 0000000..e00b5d9
--- /dev/null
+++ b/tools/jenkins-slave-creation-unix/conf-ubuntu-gpu-p3/install.sh
@@ -0,0 +1,141 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+set -x
+
+# Load variables
+source /var/lib/cloud/instance/scripts/part-001
+
+#Setup jenkins user
+sudo useradd jenkins_slave
+
+#Prevent log in
+sudo usermod -L jenkins_slave
+sudo mkdir -p /home/jenkins_slave/remoting
+sudo chown -R jenkins_slave:jenkins_slave /home/jenkins_slave
+
+#Remove preinstalled packaged
+sudo apt-get -y purge openjdk*
+sudo apt-get -y purge nvidia*
+echo "Purged packages"
+
+#Add third party repositories
+sudo add-apt-repository -y ppa:graphics-drivers/ppa
+sudo curl -fsSL https://apt.dockerproject.org/gpg | sudo apt-key add -
+sudo apt-key fingerprint 58118E89F3A912897C070ADBF76221572C52609D
+sudo add-apt-repository "deb https://apt.dockerproject.org/repo/ ubuntu-xenial main"
+sudo apt-get update
+echo "Added third party repositories"
+
+#Install htop
+sudo apt-get -y install htop
+
+#Install java
+sudo apt-get -y install openjdk-8-jre
+
+#Install git
+sudo apt-get -y install git
+sudo -H -S -u jenkins_slave git config --global user.email "mxnet-ci"
+sudo -H -S -u jenkins_slave git config --global user.name "mxnet-ci"
+
+#Install python3, pip3 and dependencies for auto-connect.py
+sudo apt-get -y install python3 python3-pip
+sudo pip3 install boto3 python-jenkins joblib docker
+
+echo "Installed htop, java, git and python"
+
+#Install nvidia drivers
+sudo apt-get -y install nvidia-410
+
+# TODO: - Disabled nvidia updates @ /etc/apt/apt.conf.d/50unattended-upgrades
+#Unattended-Upgrade::Package-Blacklist {
+#"nvidia-384";
+#"nvidia-opencl-icd-384";
+
+echo "Installed nvidia drivers"
+
+#Install docker engine
+sudo apt-get install -y apt-transport-https ca-certificates curl software-properties-common
+curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+sudo apt-key fingerprint 0EBFCD88
+sudo add-apt-repository \
+ "deb [arch=amd64] https://download.docker.com/linux/ubuntu \
+ $(lsb_release -cs) \
+ stable"
+sudo apt-get update
+sudo apt-get install -y docker-ce
+sudo usermod -aG docker jenkins_slave
+sudo systemctl enable docker #Enable docker to start on startup
+sudo service docker restart
+echo "Installed docker engine"
+
+# Add nvidia-docker and nvidia-docker-plugin
+curl -s -L https://nvidia.github.io/nvidia-docker/gpgkey | \
+ sudo apt-key add -
+distribution=$(. /etc/os-release;echo $ID$VERSION_ID)
+curl -s -L https://nvidia.github.io/nvidia-docker/$distribution/nvidia-docker.list | \
+ sudo tee /etc/apt/sources.list.d/nvidia-docker.list
+sudo apt-get update
+
+# Install nvidia-docker2 and reload the Docker daemon configuration
+sudo apt-get install -y nvidia-docker2
+sudo pkill -SIGHUP dockerd
+
+# Download additional scripts
+sudo apt-get -y install awscli
+tmpdir=$(mktemp -d)
+aws s3 cp --quiet s3://$S3_CONFIG_BUCKET/$S3_CONFIG_FILE $tmpdir/scripts.tar.bz2
+
+mkdir /home/jenkins_slave/scripts
+tar -C /home/jenkins_slave/scripts -xjf $tmpdir/scripts.tar.bz2
+find /home/jenkins_slave/scripts -type f -exec chmod 744 {} \;
+chown -R jenkins_slave.jenkins_slave /home/jenkins_slave
+
+# Set up swap of 50GB
+fallocate -l 50G /swapfile
+chown root:root /swapfile
+chmod 0600 /swapfile
+mkswap /swapfile
+swapon /swapfile
+echo /swapfile none swap sw 0 0 >> /etc/fstab
+
+# Add auto-connecting to slave to startup
+touch /home/jenkins_slave/auto-connect.log
+chown -R jenkins_slave:jenkins_slave /home/jenkins_slave/
+echo "@reboot jenkins_slave /home/jenkins_slave/scripts/launch-autoconnect.sh" > /etc/cron.d/jenkins-start-slave
+
+# Write instructions to home dir
+readme="Please use the following command in your cloud-init-script to specify the jenkins master
+address:
+#!/bin/bash
+echo 'http://jenkins.mxnet-ci.amazon-ml.com/' > /home/jenkins_slave/jenkins_master_url
+echo 'http://jenkins-priv.mxnet-ci.amazon-ml.com/' > /home/jenkins_slave/jenkins_master_private_url
+
+
+Optional:
+echo 'mxnet-linux-cpu10' > /home/jenkins_slave/jenkins_slave_name
+"
+echo "$readme" > /home/ubuntu/readme.txt
+
+echo "Setup completed"
+
+# For testing use reboot, but lateron just turn off the instance to prepare AMI generation
+# reboot
+shutdown -h now
diff --git a/tools/jenkins-slave-creation-unix/conf-ubuntu-gpu-p3/shell-variables.sh b/tools/jenkins-slave-creation-unix/conf-ubuntu-gpu-p3/shell-variables.sh
new file mode 100644
index 0000000..42e24e9
--- /dev/null
+++ b/tools/jenkins-slave-creation-unix/conf-ubuntu-gpu-p3/shell-variables.sh
@@ -0,0 +1,22 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+
+export S3_CONFIG_BUCKET="mxnet-ci-slave-dev"
+export S3_CONFIG_FILE="ubuntu-gpu-p3-config.tar.bz2"
diff --git a/tools/jenkins-slave-creation-unix/create_slave.sh b/tools/jenkins-slave-creation-unix/create_slave.sh
new file mode 100755
index 0000000..10b1f13
--- /dev/null
+++ b/tools/jenkins-slave-creation-unix/create_slave.sh
@@ -0,0 +1,26 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# See https://cwiki.apache.org/confluence/display/MXNET/Setup#Setup-Slave for instructions
+echo "Config dir:"
+read config_dir
+
+export AWS_PROFILE=mxnet-ci-dev
+
+python3 scripts/create_slave.py --configdir "$config_dir" --terraformdir "."
diff --git a/tools/jenkins-slave-creation-unix/infrastructure.tf b/tools/jenkins-slave-creation-unix/infrastructure.tf
new file mode 100644
index 0000000..3fd91a7
--- /dev/null
+++ b/tools/jenkins-slave-creation-unix/infrastructure.tf
@@ -0,0 +1,326 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+variable "key_name" {
+ type = "string"
+}
+
+variable "key_path" {
+ type = "string"
+}
+
+variable "instance_type" {
+ type = "string"
+}
+
+variable "additional_security_group_ids" {
+ type = "list"
+}
+
+variable "secret_manager_docker_hub_arn" {
+ type = "string"
+}
+
+variable "s3_config_bucket" {
+ type = "string"
+}
+
+variable "s3_config_filename" {
+ type = "string"
+}
+
+variable "slave_install_script" {
+ type = "string"
+}
+
+variable "shell_variables_file" {
+ type = "string"
+}
+
+# ftp://64.50.236.216/pub/ubuntu-cloud-images/query/xenial/server/released.txt
+# https://cloud-images.ubuntu.com/locator/ec2/
+variable "ami" {
+ type = "string"
+}
+
+variable "instance_name" {
+ type = "string"
+}
+
+variable "aws_region" {
+ type = "string"
+}
+
+# Input variables (done by script)
+variable "slave_config_tar_path" {
+ type = "string"
+}
+
+provider "aws" {
+ region = "${var.aws_region}"
+}
+
+# Store terraform state in S3 instead of local.
+terraform {
+ backend "s3" {
+ encrypt = true
+ key = "terraform.tfstate"
+ # Remaining config is defined in $CONFIG_DIR/infrastructure_backend.tfvars
+ # See https://www.terraform.io/docs/backends/config.html for more details
+ }
+}
+
+data "template_cloudinit_config" "user_data" {
+ base64_encode = true
+
+ # Important: This part has to be in first place as it gets mapped to
+ # /var/lib/cloud/instance/scripts/part-001
+ # This is a hack, but there's no other way to reference other scripts
+ part {
+ content_type = "text/x-shellscript"
+ content = "${file("${var.shell_variables_file}")}"
+ }
+
+
+ part {
+ content_type = "text/x-shellscript"
+ content = "${file("${var.slave_install_script}")}"
+ }
+}
+
+resource "aws_iam_role" "jenkins_slave_role" {
+ name = "jenkins_slave_role"
+ assume_role_policy = <<POLICY
+{
+ "Version": "2012-10-17",
+ "Statement": [
+ {
+ "Action": "sts:AssumeRole",
+ "Principal": {
+ "Service": "ec2.amazonaws.com"
+ },
+ "Effect": "Allow",
+ "Sid": ""
+ }
+ ]
+}
+POLICY
+}
+
+resource "aws_iam_role" "jenkins_restricted_slave_role" {
+ name = "jenkins_restricted_slave_role"
+ assume_role_policy = <<POLICY
+{
+ "Version": "2012-10-17",
+ "Statement": [
+ {
+ "Action": "sts:AssumeRole",
+ "Principal": {
+ "Service": "ec2.amazonaws.com"
+ },
+ "Effect": "Allow",
+ "Sid": ""
+ }
+ ]
+}
+POLICY
+}
+
+
+resource "aws_iam_policy" "jenkins_slave_s3_read_policy" {
+ name = "jenkins_slave_s3_read_policy"
+ description = "Policy to grant Jenkins Slave S3 read-access to the associated bucket"
+ policy = <<POLICY
+{
+ "Version": "2012-10-17",
+ "Statement": [
+ {
+ "Effect": "Allow",
+ "Action": [
+ "s3:GetBucketLocation",
+ "s3:ListAllMyBuckets"
+ ],
+ "Resource": "arn:aws:s3:::*"
+ },
+ {
+ "Effect": "Allow",
+ "Action": [
+ "s3:ListBucket"
+ ],
+ "Resource": [
+ "arn:aws:s3:::${var.s3_config_bucket}"
+ ]
+ },
+ {
+ "Effect": "Allow",
+ "Action": [
+ "s3:GetObject"
+ ],
+ "Resource": [
+ "arn:aws:s3:::${var.s3_config_bucket}/*"
+ ]
+ },
+ {
+ "Effect": "Allow",
+ "Action": [
+ "ec2:CreateTags"
+ ],
+ "Resource": [
+ "arn:aws:s3:::instance/*"
+ ]
+ }
+
+
+ ]
+}
+POLICY
+}
+
+
+resource "aws_iam_policy_attachment" "jenkins_slave_s3_read_policy_attach" {
+ name = "jenkins_slave_s3_read_policy_attach"
+ roles = [
+ "${aws_iam_role.jenkins_slave_role.name}",
+ "${aws_iam_role.jenkins_restricted_slave_role.name}"
+ ]
+ policy_arn = "${aws_iam_policy.jenkins_slave_s3_read_policy.arn}"
+}
+
+resource "aws_iam_policy" "jenkins_slave_ec2_create_tags" {
+ name = "jenkins_slave_ec2_create_tags"
+ description = "Policy to grant Jenkins Slave permission to create tags in order to rename an instance"
+ policy = <<POLICY
+{
+ "Version": "2012-10-17",
+ "Statement": [
+ {
+ "Effect": "Allow",
+ "Action": [
+ "ec2:CreateTags"
+ ],
+ "Resource": [
+ "arn:aws:ec2:*:*:instance/*"
+ ]
+ }
+
+
+ ]
+}
+POLICY
+}
+
+resource "aws_iam_policy_attachment" "jenkins_slave_ec2_create_tags_attach" {
+ name = "jenkins_slave_ec2_create_tags_attach"
+ roles = [
+ "${aws_iam_role.jenkins_slave_role.name}",
+ "${aws_iam_role.jenkins_restricted_slave_role.name}"
+ ]
+ policy_arn = "${aws_iam_policy.jenkins_slave_ec2_create_tags.arn}"
+}
+
+resource "aws_iam_policy" "jenkins_restricted_slave_secrets_docker_cache" {
+ name = "jenkins_restricted_slave_secrets_docker_cache"
+ description = "Policy to grant restricted Jenkins Slave permission to the Secret Manager to do Docker Hub credential retrieval for distributed Docker cache"
+ policy = <<POLICY
+{
+ "Version": "2012-10-17",
+ "Statement": [
+ {
+ "Effect": "Allow",
+ "Action": [
+ "secretsmanager:DescribeSecret",
+ "secretsmanager:List*"
+ ],
+ "Resource": "*"
+ },
+ {
+ "Effect": "Allow",
+ "Action": "secretsmanager:*",
+ "Resource": [
+ "${var.secret_manager_docker_hub_arn}"
+ ]
+ }
+ ]
+}
+POLICY
+}
+
+# Only apply this permission to restricted slaves!!!
+resource "aws_iam_policy_attachment" "jenkins_restricted_slave_secrets_docker_cache_attach" {
+ name = "jenkins_restricted_slave_secrets_docker_cache_attach"
+ roles = ["${aws_iam_role.jenkins_restricted_slave_role.name}"]
+ policy_arn = "${aws_iam_policy.jenkins_restricted_slave_secrets_docker_cache.arn}"
+}
+
+resource "aws_iam_instance_profile" "jenkins_slave_profile" {
+ name = "jenkins_slave_profile"
+ role = "${aws_iam_role.jenkins_slave_role.name}"
+}
+
+resource "aws_iam_instance_profile" "jenkins_restricted_slave_profile" {
+ name = "jenkins_restricted_slave_profile"
+ role = "${aws_iam_role.jenkins_restricted_slave_role.name}"
+}
+
+resource "aws_instance" "mxnet-slave" {
+ instance_type = "${var.instance_type}"
+ ami = "${var.ami}"
+ iam_instance_profile = "${aws_iam_instance_profile.jenkins_slave_profile.name}"
+
+ # The name of our SSH keypair you've created and downloaded
+ # from the AWS console.
+ #
+ # https://console.aws.amazon.com/ec2/v2/home?region=us-west-2#KeyPairs:
+ #
+ key_name = "${var.key_name}"
+
+ vpc_security_group_ids = ["${
+ concat(
+ var.additional_security_group_ids
+ )
+ }"]
+
+ user_data = "${data.template_cloudinit_config.user_data.rendered}"
+
+ tags {
+ "Name" = "${var.instance_name}"
+ }
+
+
+ root_block_device {
+ volume_type = "gp2"
+ volume_size = 350
+ delete_on_termination = true
+ }
+
+ # Wait for S3 bucket as it's needed during startup
+ depends_on = [
+ "aws_s3_bucket_object.slave_config_s3"
+ ]
+}
+
+resource "aws_s3_bucket" "slave_config_bucket" {
+ bucket = "${var.s3_config_bucket}"
+ acl = "private"
+}
+
+resource "aws_s3_bucket_object" "slave_config_s3" {
+ bucket = "${aws_s3_bucket.slave_config_bucket.id}"
+ key = "${var.s3_config_filename}"
+ source = "${var.slave_config_tar_path}"
+ etag = "${md5(file(var.slave_config_tar_path))}"
+}
diff --git a/tools/jenkins-slave-creation-unix/init.sh b/tools/jenkins-slave-creation-unix/init.sh
new file mode 100755
index 0000000..0c01886
--- /dev/null
+++ b/tools/jenkins-slave-creation-unix/init.sh
@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+virtualenv -p `which python3` py3env
+source py3env/bin/activate
+pip install python-terraform
+
diff --git a/tools/jenkins-slave-creation-unix/scripts/create_slave.py b/tools/jenkins-slave-creation-unix/scripts/create_slave.py
new file mode 100644
index 0000000..7e1988d
--- /dev/null
+++ b/tools/jenkins-slave-creation-unix/scripts/create_slave.py
@@ -0,0 +1,106 @@
+#!/usr/bin/env python3
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# -*- coding: utf-8 -*-
+
+# This script serves the purpose to automatize the creation process of a jenkins slave
+
+import logging
+import argparse
+import os
+import re
+import time
+import tarfile
+import pprint
+import shutil
+from python_terraform import *
+from tempfile import TemporaryDirectory
+from shutil import copytree
+
+
+
+TERRAFORM_SCRIPT_NAME = 'infrastructure.tf'
+TERRAFORM_VARFILE_NAME = 'infrastructure.tfvars'
+TERRAFORM_BACKEND_VARFILE_NAME = 'infrastructure_backend.tfvars'
+
+
+DEPLOYED_SCRIPTS_DIR_NAME = 'scripts/deploy'
+
+
+def main():
+ logging.getLogger().setLevel(logging.DEBUG)
+
+ parser = argparse.ArgumentParser()
+ parser.add_argument('-d', '--configdir',
+ help='Deployment configuration directory',
+ default='conf-ubuntu-cpu',
+ type=str)
+
+ parser.add_argument('-tf', '--terraformdir',
+ help='Directory containing the terraform scripts',
+ default='.',
+ type=str)
+
+ args = parser.parse_args()
+
+ terraform_dir_abs = os.path.abspath(args.terraformdir)
+
+ if not os.path.isfile(os.path.join(terraform_dir_abs, TERRAFORM_SCRIPT_NAME)):
+ raise FileNotFoundError('Unable to find terraform script. Did you specify "--terraformdir"? {}'.
+ format(os.path.join(terraform_dir_abs, TERRAFORM_SCRIPT_NAME)))
+
+ # Copy configuration to temp dir
+ with TemporaryDirectory() as temp_dir:
+ temp_archive_path = os.path.join(temp_dir, "config.tar.bz2")
+
+ with tarfile.open(temp_archive_path, "w:bz2") as tar:
+ scripts_dir = os.path.join(args.terraformdir, DEPLOYED_SCRIPTS_DIR_NAME)
+ for file in os.listdir(scripts_dir):
+ logging.debug('Archiving {}'.format(file))
+ tar.add(os.path.join(scripts_dir, file), arcname=os.path.basename(file))
+
+ # Trigger terraform
+ logging.info('Running terraform...')
+
+ logging.debug('Switching current work dir to {}'.format(terraform_dir_abs))
+ os.chdir(terraform_dir_abs)
+
+ # Setting up the terraform S3 backend requires to have AWS credentials in the env vars - it's not able
+ # to access the variables file due interpolation in terraform being enabled after initialization of
+ # the s3 backend
+ env_vars = os.environ.copy()
+
+ subprocess.check_call('~/bin/terraform init -backend-config={}'.format(os.path.join(args.configdir, TERRAFORM_BACKEND_VARFILE_NAME)), cwd=terraform_dir_abs, env=env_vars, shell=True)
+ command = input("Terraform apply or plan? [apply]: ") or "apply"
+ subprocess.check_call('~/bin/terraform {} -var-file="{}" -var "slave_config_tar_path={}"'.format(command, os.path.join(args.configdir, TERRAFORM_VARFILE_NAME), temp_archive_path), cwd=terraform_dir_abs, env=env_vars, shell=True)
+ logging.debug('Deployment finished')
+
+
+def get_tfvars_entry(tfvars_file, key):
+ # This is just a hack because I don't want to spend the time to write an entire parser for the .tfvars format
+ with open(tfvars_file, 'r') as fp:
+ for line in fp:
+ if line.startswith(key):
+ result = re.search('"(.*)"', line).group(1)
+ return result
+
+ raise ValueError('Could not find {} in {}'.format(key, tfvars_file))
+
+if __name__ == '__main__':
+ main()
diff --git a/tools/jenkins-slave-creation-unix/scripts/deploy/launch-autoconnect.sh b/tools/jenkins-slave-creation-unix/scripts/deploy/launch-autoconnect.sh
new file mode 100644
index 0000000..4da33a6
--- /dev/null
+++ b/tools/jenkins-slave-creation-unix/scripts/deploy/launch-autoconnect.sh
@@ -0,0 +1,22 @@
+#!/bin/sh
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -ex
+
+python3 /home/jenkins_slave/scripts/slave-autoconnect.py --slave-name-file=/home/jenkins_slave/jenkins_slave_name --master-file=/home/jenkins_slave/jenkins_master_url --master-private-file=/home/jenkins_slave/jenkins_master_private_url > /home/jenkins_slave/auto-connect.log
diff --git a/tools/jenkins-slave-creation-unix/scripts/deploy/launch-ccache-mount-efs.sh b/tools/jenkins-slave-creation-unix/scripts/deploy/launch-ccache-mount-efs.sh
new file mode 100644
index 0000000..15190ee
--- /dev/null
+++ b/tools/jenkins-slave-creation-unix/scripts/deploy/launch-ccache-mount-efs.sh
@@ -0,0 +1,40 @@
+#!/bin/sh
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -ex
+mkdir /efs-ccache
+
+# Wait until config file is available (on startup)
+EFS_CONFIG_FILE_PATH=/home/jenkins_slave/ccache_efs_address
+MAX_TIMEOUT=300
+timeout_counter=0
+while [ "$timeout_counter" -lt $MAX_TIMEOUT -a ! -e $EFS_CONFIG_FILE_PATH ]; do
+ sleep 1
+ timeout_counter=$((timeout_counter+1))
+done
+if [ -e $EFS_CONFIG_FILE_PATH ]
+then
+ EFS_DNS=`cat $EFS_CONFIG_FILE_PATH`
+ echo "Found $EFS_CONFIG_FILE_PATH with content $EFS_DNS after $timeout_counter seconds"
+ sudo mount -t nfs4 -o nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2 $EFS_DNS:/ /efs-ccache
+ sudo chown jenkins_slave:jenkins_slave /efs-ccache
+else
+ echo "Timeout looking for $EFS_CONFIG_FILE_PATH" >&2
+fi
+
diff --git a/tools/jenkins-slave-creation-unix/scripts/deploy/slave-autoconnect.py b/tools/jenkins-slave-creation-unix/scripts/deploy/slave-autoconnect.py
new file mode 100644
index 0000000..54b909f
--- /dev/null
+++ b/tools/jenkins-slave-creation-unix/scripts/deploy/slave-autoconnect.py
@@ -0,0 +1,342 @@
+#!/usr/bin/env python3
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import platform
+import logging
+import jenkins
+import functools
+import argparse
+import os
+import urllib.request
+import pprint
+import subprocess
+import sys
+import shutil
+import boto3
+import time
+import re
+import random
+
+AGENT_SLAVE_JAR_PATH = 'jnlpJars/slave.jar'
+LOCAL_SLAVE_JAR_PATH = 'slave.jar'
+SLAVE_CONNECTION_URL_FORMAT = "{master_private}/computer/{label}/slave-agent.jnlp"
+SLAVE_START_COMMAND = 'java -jar {slave_path} -jnlpUrl {connection_url} -workDir "{work_dir}" -failIfWorkDirIsMissing'
+REGION_REGEX = r'"region" : "([a-z0-9\-]+)"'
+INSTANCE_ID_REGEX = r'"instanceId" : "([a-z0-9\-]+)"'
+RETRY_COUNTER_RESET_TIME_SECONDS = 600
+RETRY_LIMIT = 5
+RETRY_SLEEP_MIN_SECONDS = 5
+RETRY_SLEEP_MAX_SECONDS = 30
+CLOUD_INIT_MAX_WAIT_SECONDS = 120
+
+
+"""
+In order to use this script, it has to be placed in auto start of your instance. Put the following into the user-data
+part of the launch command in order to get the parameters initialized:
+
+#!/bin/bash
+echo 'http://jenkins.mxnet-ci-dev.amazon-ml.com/' > /home/jenkins_slave/jenkins_master_url
+echo 'http://jenkins-priv.mxnet-ci-dev.amazon-ml.com/' > /home/jenkins_slave/jenkins_master_private_url
+echo 'mxnet-linux-cpu10' > /home/jenkins_slave/jenkins_slave_name
+"""
+
+
+def connect_to_master(node_name, master_private_url, work_dir):
+ # We have to rename this instance before it is able to connect because we're not getting back the control
+ # if the launch was successful
+ rename_instance(node_name)
+
+ # Try to connect to this node. If it fails, there's probably already a node connected to that slot
+ slave_connection_url = SLAVE_CONNECTION_URL_FORMAT.format(master_private=master_private_url, label=node_name)
+ slave_start_command = SLAVE_START_COMMAND.format(connection_url=slave_connection_url, work_dir=work_dir,
+ slave_path=LOCAL_SLAVE_JAR_PATH)
+ logging.info('slave start command: {}'.format(slave_start_command))
+ return_code = subprocess.call(slave_start_command, shell=True)
+ # TODO: Wait if this line appears: 'is already connected to this master. Rejecting this connection.'
+ return return_code
+
+
+def download(url: str, file: str):
+ logging.debug('Downloading {} to {}'.format(url, file))
+ urllib.request.urlretrieve(url, file)
+
+
+def is_offline_node_matches_prefix(prefix: str, node) -> bool:
+ return node['name'].startswith(prefix) and node['offline']
+
+
+def generate_node_label():
+ system = platform.system()
+ labelPlatform = "mxnet-"
+
+ # Determine platform type
+ if system == "Windows":
+ labelPlatform += "windows-"
+ elif system == "Linux":
+ labelPlatform += "linux-"
+ else:
+ raise RuntimeError("system {} is not supported yet".format(system))
+
+ # Determine whether CPU or GPU system
+ if is_gpu_present():
+ labelPlatform += "gpu"
+ else:
+ labelPlatform += "cpu"
+
+ return labelPlatform
+
+def rename_instance(name: str):
+ logging.info('Renaming instance to {}'.format(name))
+ response = urllib.request.urlopen("http://169.254.169.254/latest/dynamic/instance-identity/document")
+ instance_info = response.read().decode('utf-8')
+ logging.debug('Instance info: {}'.format(instance_info))
+
+ region_match = re.search(REGION_REGEX, instance_info)
+ instance_id_match = re.search(INSTANCE_ID_REGEX, instance_info)
+
+ if not region_match or not instance_id_match:
+ raise RuntimeError("Unable to determine instance id or region. Instance info: {}".format(instance_info))
+
+ region = region_match.group(1)
+ instance_id = instance_id_match.group(1)
+
+ logging.debug('Instance region: {} Instance id: {}'.format(region, instance_id))
+
+ ec2 = boto3.resource('ec2', region_name=region)
+ ec2.create_tags(
+ DryRun=False,
+ Resources=[
+ instance_id
+ ],
+ Tags=[
+ {
+ 'Key': 'Name',
+ 'Value': name
+ },
+ ]
+ )
+
+
+def is_gpu_present() -> bool:
+ num_gpus = get_num_gpus()
+ logging.debug('Number GPUs present: {}'.format(num_gpus))
+ return num_gpus > 0
+
+
+def get_num_gpus() -> int:
+ """
+ Gets the number of GPUs available on the host (depends on nvidia-smi).
+ :return: The number of GPUs on the system.
+ """
+ #if shutil.which("nvidia-smi") is None:
+ nvidia_smi_path = get_nvidia_smi_path()
+ if nvidia_smi_path is None or shutil.which(nvidia_smi_path) is None:
+ logging.warning("Couldn't find nvidia-smi, therefore we assume no GPUs are available.")
+ return 0
+ #sp = subprocess.Popen(['nvidia-smi', '-L'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ sp = subprocess.Popen([get_nvidia_smi_path(), '-L'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+
+ out_str = sp.communicate()[0].decode("utf-8")
+
+ # Ensure we're counting the lines with GPU as CPU-instances have nvidia-smi present as well
+ num_gpus = 0
+ for line in out_str.split("\n"):
+ logging.debug('Nvidia-SMI: {}'.format(line))
+ if 'GPU' in line:
+ num_gpus += 1
+
+ return num_gpus
+
+
+def get_nvidia_smi_path() -> str:
+ if shutil.which("nvidia-smi") is not None:
+ return 'nvidia-smi'
+
+ if shutil.which("C:\\Program Files\\NVIDIA Corporation\\NVSMI\\nvidia-smi.exe") is not None:
+ return 'C:\\Program Files\\NVIDIA Corporation\\NVSMI\\nvidia-smi.exe'
+
+ return None
+
+
+def read_master_urls(args) -> (str,str):
+ try:
+ if args.master is not None:
+ master = args.master
+ else:
+ if args.master_file is not None:
+ master = read_file_content(file_path=args.master_file,
+ timeout_seconds=CLOUD_INIT_MAX_WAIT_SECONDS)
+ if not master:
+ raise ValueError('{} is empty'.format(args.master_file))
+ else:
+ raise ValueError('--master_file not specified')
+
+ if args.master_private is not None:
+ master_private = args.master_private
+ else:
+ if args.master_private_file is not None:
+ master_private = read_file_content(file_path=args.master_private_file,
+ timeout_seconds=CLOUD_INIT_MAX_WAIT_SECONDS)
+ if not master_private:
+ raise ValueError('{} is empty'.format(args.master_private_file))
+ else:
+ raise ValueError('--master_private_file not specified')
+
+ logging.debug('Master: {} Private: {}'.format(master, master_private))
+ return master, master_private
+ except:
+ logging.exception('Error during reading master URLs')
+ rename_instance('read-master-config-error')
+ raise
+
+
+def read_name_from_path(name_path):
+ if name_path is None:
+ return None
+
+ return read_file_content(file_path=name_path, timeout_seconds=120)
+
+
+def read_file_content(file_path, timeout_seconds):
+ """
+ Read file content
+ :param path: Filepath
+ :param timeout_seconds: Time to wait until file exists
+ :return: File content
+ """
+ end_time = time.time() + timeout_seconds
+ while end_time > time.time() and not os.path.exists(file_path):
+ time.sleep(1)
+
+ if end_time <= time.time():
+ raise FileNotFoundError('Timeout waiting for file {} to exist'.format(file_path))
+
+ with open(file_path, "r") as file_handle:
+ content = file_handle.readline().strip()
+ if not content:
+ raise ValueError('{} is empty'.format(file_path))
+
+ return content
+
+
+def main():
+ try:
+ logging.getLogger().setLevel(logging.DEBUG)
+ parser = argparse.ArgumentParser()
+ parser.add_argument('-m', '--master',
+ help='URL of jenkins master',
+ # default='http://jenkins.mxnet-ci.amazon-ml.com',
+ type=str)
+
+ parser.add_argument('-mf', '--master-file',
+ help='File containing URL of jenkins master',
+ # default='/home/jenkins_slave/jenkins_master_url',
+ type=str)
+
+ parser.add_argument('-mp', '--master-private',
+ help='Private URL of jenkins master',
+ # default='http://jenkins-priv.mxnet-ci.amazon-ml.com',
+ type=str)
+
+ parser.add_argument('-mpf', '--master-private-file',
+ help='File containing private URL of jenkins master',
+ # default='/home/jenkins_slave/jenkins_master_private_url',
+ type=str)
+
+ parser.add_argument('-snf', '--slave-name-file',
+ help='File containing name of the slave slot',
+ type=str)
+
+ args = parser.parse_args()
+
+ master_url, master_private_url = read_master_urls(args)
+ slave_name = read_name_from_path(args.slave_name_file)
+
+ # Replace \ by / on URL due to windows using \ as default separator
+ jenkins_slave_jar_url = os.path.join(master_url, AGENT_SLAVE_JAR_PATH).replace('\\', '/')
+
+ # Download jenkins slave jar
+ download(jenkins_slave_jar_url, LOCAL_SLAVE_JAR_PATH)
+
+ work_dir = os.path.join(os.getcwd(), 'workspace')
+ logging.info('Work dir: {}'.format(work_dir))
+
+ # Create work dir if it doesnt exist
+ os.makedirs(work_dir, exist_ok=True)
+ os.makedirs(os.path.join(work_dir, 'remoting'), exist_ok=True)
+
+ server = jenkins.Jenkins(master_url)
+
+ i = 0
+ while i < RETRY_LIMIT:
+ i += 1
+ if slave_name:
+ logging.info('Entering manual connect mode to slave slot {}'.format(slave_name))
+ offline_nodes = [slave_name]
+ else:
+ logging.info('Entering auto connect mode')
+ label = generate_node_label()
+ logging.info('Local node prefix: {}'.format(label))
+ nodes = server.get_nodes()
+
+ offline_nodes = [node['name'] for node in
+ list(filter(functools.partial(is_offline_node_matches_prefix, label), nodes))]
+ logging.debug('Offline nodes: {}', offline_nodes)
+ # Shuffle to provide random order to reduce race conditions if multiple instances
+ # are started at the same time and thus try to connect to the same slot, possibly
+ # resulting in a hang
+ random.shuffle(offline_nodes)
+
+ if len(offline_nodes) == 0:
+ rename_instance('error-no-free-slot')
+ logging.fatal('Could connect to master - no free slots')
+ return 1
+
+ reset = False
+ # Loop through nodes and try to connect
+ for node_name in offline_nodes:
+ start_time = time.time()
+ connect_to_master(node_name=node_name, master_private_url=master_private_url, work_dir=work_dir)
+ total_runtime_seconds = time.time() - start_time
+
+ if total_runtime_seconds > RETRY_COUNTER_RESET_TIME_SECONDS:
+ logging.info('Instance ran for {} seconds, resetting retry counter'.format(total_runtime_seconds))
+ reset = True
+ else:
+ logging.info('Unable to connect as node {}'.format(node_name))
+
+ # Rename this instance to show it was unable to connect
+ rename_instance('{}-unable-to-connect'.format(node_name))
+
+ time.sleep(random.randint(RETRY_SLEEP_MIN_SECONDS, RETRY_SLEEP_MAX_SECONDS))
+ if reset:
+ logging.info('Resetting repetition counter')
+ i = 0
+
+ rename_instance('error-too-many-attempts')
+ logging.fatal('Could connect to master - too many attempts')
+ return 1
+ except Exception as e:
+ logging.exception('Fatal exception')
+ logging.fatal('Fatal exception, aborting execution')
+ return 1
+
+
+if __name__ == '__main__':
+ sys.exit(main())