You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by ma...@apache.org on 2019/08/15 21:42:04 UTC
[incubator-mxnet-ci] 01/01: Add Jenkins Autoscaling
This is an automated email from the ASF dual-hosted git repository.
marcoabreu pushed a commit to branch autoscaling
in repository https://gitbox.apache.org/repos/asf/incubator-mxnet-ci.git
commit a523aa546c1619985d4e6a6d5a12fa5862a1e86e
Author: Marco de Abreu <ma...@gmail.com>
AuthorDate: Thu Aug 15 23:38:15 2019 +0200
Add Jenkins Autoscaling
---
services/jenkins-autoscaling/README.md | 20 +
services/jenkins-autoscaling/kill_autoscaled.py | 37 +
.../jenkins-autoscaling/lambda_mxnet_ci/.gitignore | 108 ++
.../lambda_mxnet_ci/__init__.py | 0
.../lambda_mxnet_ci/autoscaling/__init__.py | 0
.../lambda_mxnet_ci/autoscaling/attach-log.sh | 20 +
.../autoscaling/cw_event_timeout.sh | 21 +
.../lambda_mxnet_ci/autoscaling/deploy-lambda.sh | 35 +
.../lambda_mxnet_ci/autoscaling/environment.yml | 47 +
.../lambda_mxnet_ci/autoscaling/handler.py | 1533 ++++++++++++++++++++
.../lambda_mxnet_ci/autoscaling/requirements.txt | 4 +
.../lambda_mxnet_ci/autoscaling/serverless.yml | 53 +
.../jenkins-autoscaling/lambda_mxnet_ci/pylintrc | 415 ++++++
.../lambda_mxnet_ci/tests/.coveragerc | 44 +
.../lambda_mxnet_ci/tests/.gitignore | 3 +
.../ec2.DescribeInstances_1.json | 361 +++++
.../ec2.DescribeInstances_1.json | 520 +++++++
.../ec2.DescribeInstances_1.json | 294 ++++
.../ec2.DescribeInstances_2.json | 294 ++++
.../ec2.TerminateInstances_1.json | 63 +
.../lambda_mxnet_ci/tests/test_autoscaling.py | 1258 ++++++++++++++++
services/jenkins-autoscaling/requirements.txt | 30 +
22 files changed, 5160 insertions(+)
diff --git a/services/jenkins-autoscaling/README.md b/services/jenkins-autoscaling/README.md
new file mode 100644
index 0000000..581bde8
--- /dev/null
+++ b/services/jenkins-autoscaling/README.md
@@ -0,0 +1,20 @@
+<!--- Licensed to the Apache Software Foundation (ASF) under one -->
+<!--- or more contributor license agreements. See the NOTICE file -->
+<!--- distributed with this work for additional information -->
+<!--- regarding copyright ownership. The ASF licenses this file -->
+<!--- to you under the Apache License, Version 2.0 (the -->
+<!--- "License"); you may not use this file except in compliance -->
+<!--- with the License. You may obtain a copy of the License at -->
+
+<!--- http://www.apache.org/licenses/LICENSE-2.0 -->
+
+<!--- Unless required by applicable law or agreed to in writing, -->
+<!--- software distributed under the License is distributed on an -->
+<!--- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->
+<!--- KIND, either express or implied. See the License for the -->
+<!--- specific language governing permissions and limitations -->
+<!--- under the License. -->
+
+Autoscaling system for Jenkins
+
+See https://cwiki.apache.org/confluence/display/MXNET/Setup#Setup-Autoscaling for documentation
diff --git a/services/jenkins-autoscaling/kill_autoscaled.py b/services/jenkins-autoscaling/kill_autoscaled.py
new file mode 100755
index 0000000..105fa82
--- /dev/null
+++ b/services/jenkins-autoscaling/kill_autoscaled.py
@@ -0,0 +1,37 @@
+#!/usr/bin/env python3
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# -*- coding: utf-8 -*-
+import sys
+def main():
+ import boto3
+ ses = boto3.Session()
+ ec2 = ses.resource('ec2', region_name='us-west-2')
+ iif= ec2.instances.filter(
+ #Filters=[{'Name': 'tag:Name', 'Values': ['error-too-many-attempts']}
+ Filters=[{'Name': 'tag:AutoScaledSlave', 'Values': ['True']}
+ ])
+ print(list(iif))
+ #iif.terminate()
+ return 1
+
+if __name__ == '__main__':
+ sys.exit(main())
+
+
diff --git a/services/jenkins-autoscaling/lambda_mxnet_ci/.gitignore b/services/jenkins-autoscaling/lambda_mxnet_ci/.gitignore
new file mode 100644
index 0000000..078e972
--- /dev/null
+++ b/services/jenkins-autoscaling/lambda_mxnet_ci/.gitignore
@@ -0,0 +1,108 @@
+node_modules
+package-lock.json
+.serverless/
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+# Usually these files are written by a python script from a template
+# before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
diff --git a/services/jenkins-autoscaling/lambda_mxnet_ci/__init__.py b/services/jenkins-autoscaling/lambda_mxnet_ci/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/services/jenkins-autoscaling/lambda_mxnet_ci/autoscaling/__init__.py b/services/jenkins-autoscaling/lambda_mxnet_ci/autoscaling/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/services/jenkins-autoscaling/lambda_mxnet_ci/autoscaling/attach-log.sh b/services/jenkins-autoscaling/lambda_mxnet_ci/autoscaling/attach-log.sh
new file mode 100755
index 0000000..2f946fa
--- /dev/null
+++ b/services/jenkins-autoscaling/lambda_mxnet_ci/autoscaling/attach-log.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+serverless logs -f mxnet-ci-master-autoscaling
diff --git a/services/jenkins-autoscaling/lambda_mxnet_ci/autoscaling/cw_event_timeout.sh b/services/jenkins-autoscaling/lambda_mxnet_ci/autoscaling/cw_event_timeout.sh
new file mode 100644
index 0000000..9183b98
--- /dev/null
+++ b/services/jenkins-autoscaling/lambda_mxnet_ci/autoscaling/cw_event_timeout.sh
@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# AWS console has a bug, can't change the rate
+aws --profile mxnet-ci-dev events put-rule --name mxnet-ci-master-scaling-t-MxnetDashciDashmasterDas-842YA03483CL --schedule-expression "rate(5 minutes)"
diff --git a/services/jenkins-autoscaling/lambda_mxnet_ci/autoscaling/deploy-lambda.sh b/services/jenkins-autoscaling/lambda_mxnet_ci/autoscaling/deploy-lambda.sh
new file mode 100755
index 0000000..03c52b4
--- /dev/null
+++ b/services/jenkins-autoscaling/lambda_mxnet_ci/autoscaling/deploy-lambda.sh
@@ -0,0 +1,35 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+
+echo "Deployment stage (test, prod)"
+read config_dir
+
+if [ "$config_dir" == "test" ]; then
+ echo "Deploying to test"
+ export AWS_PROFILE=mxnet-ci-dev
+ sls deploy -s test
+elif [ "$config_dir" == "prod" ]; then
+ echo "Deploying to prod"
+ export AWS_PROFILE=mxnet-ci
+ sls deploy -s prod
+else
+ echo "Unrecognized stage: ${config_dir}"
+fi
diff --git a/services/jenkins-autoscaling/lambda_mxnet_ci/autoscaling/environment.yml b/services/jenkins-autoscaling/lambda_mxnet_ci/autoscaling/environment.yml
new file mode 100644
index 0000000..16375d6
--- /dev/null
+++ b/services/jenkins-autoscaling/lambda_mxnet_ci/autoscaling/environment.yml
@@ -0,0 +1,47 @@
+test:
+ SECRET_NAME: REDACTED
+ SECRET_ENDPOINT_URL: https://secretsmanager.us-west-2.amazonaws.com
+ SECRET_ENDPOINT_REGION: us-west-2
+ EC2_ACCOUNT_ARN: arn:aws:ec2:us-west-2:REDACTED:*
+ IAM_JENKINS_SLAVE_ROLE: arn:aws:iam::REDACTED:role/jenkins_slave_role
+ IAM_JENKINS_RESTRICTED_SLAVE_ROLE: arn:aws:iam::REDACTED:role/jenkins_restricted_slave_role
+ SECRETS_MANAGER_ARN: arn:aws:secretsmanager:us-west-2:REDACTED:secret:REDACTED
+ JENKINS_PRIV_TUNNEL: jenkins-priv.mxnet-ci-dev.amazon-ml.com:48593
+ LAUNCH_TEMPLATES: '{"mxnetlinux-cpu":{"id":"lt-06a15945813ad44f2","version":"14"},"restricted-mxnetlinux-cpu":{"id":"lt-0dc74292f7d647ac6","version":"9"},"mxnetlinux-gpu":{"id":"lt-0c22f238c0edb58ab","version":"19"},"mxnetlinux-gpu-p3":{"id":"lt-00c83ee5d7aeaf4ab","version":"12"},"restricted-mxnetlinux-gpu-p3":{"id":"lt-0f893d7f3f2660c1c","version":"3"},"mxnetlinux-gpu-p3-8xlarge":{"id":"lt-0277305ae5f49782b","version":"6"},"mxnetwindows-cpu":{"id":"lt-09dff2fff6b5586f0","version":"1 [...]
+ EXECUTORS_PER_LABEL: '{"mxnetlinux-cpu":3,"restricted-mxnetlinux-cpu":3,"mxnetlinux-gpu":1,"mxnetlinux-gpu-p3":1,"restricted-mxnetlinux-gpu-p3":1,"mxnetlinux-gpu-p3-8xlarge":1,"mxnetwindows-cpu":4,"mxnetwindows-gpu":1,"utility":30,"restricted-utility":30, "restricted-mxnetlinux-gpu": 1}'
+ WARM_POOL_SIZE: '{"mxnetlinux-cpu":1,"restricted-mxnetlinux-cpu":0,"mxnetlinux-gpu":0,"mxnetlinux-gpu-p3":0,"restricted-mxnetlinux-gpu-p3":0,"mxnetlinux-gpu-p3-8xlarge":0,"mxnetwindows-cpu":1,"mxnetwindows-gpu":0,"utility":1,"restricted-utility":1, "restricted-mxnetlinux-gpu": 0}'
+ MINIMUM_QUEUE_TIMES_SEC: '{"mxnetlinux-cpu":30,"restricted-mxnetlinux-cpu":30,"mxnetlinux-gpu":30,"mxnetlinux-gpu-p3":30,"restricted-mxnetlinux-gpu-p3":30,"mxnetlinux-gpu-p3-8xlarge":30,"mxnetwindows-cpu":30,"mxnetwindows-gpu":30,"utility":3,"restricted-utility":3, "restricted-mxnetlinux-gpu": 30}'
+ CCACHE_EFS_DNS: '{"mxnetlinux-cpu":"fs-REDACTED.efs.us-west-2.amazonaws.com","restricted-mxnetlinux-cpu":"fs-REDACTED.efs.us-west-2.amazonaws.com","mxnetlinux-gpu":"NONE","mxnetlinux-gpu-p3":"NONE","restricted-mxnetlinux-gpu-p3":"NONE","mxnetlinux-gpu-p3-8xlarge":"NONE","mxnetwindows-cpu":"NONE","mxnetwindows-gpu":"NONE","utility":"NONE","restricted-utility":"NONE", "restricted-mxnetlinux-gpu": "NONE"}'
+ MAXIMUM_STARTUP_TIME_SEC: '{"mxnetlinux-cpu":300,"restricted-mxnetlinux-cpu":300,"mxnetlinux-gpu":300,"mxnetlinux-gpu-p3":300,"restricted-mxnetlinux-gpu-p3":300,"mxnetlinux-gpu-p3-8xlarge":300,"mxnetwindows-cpu":1800,"mxnetwindows-gpu":1800,"utility":300,"restricted-utility":300, "restricted-mxnetlinux-gpu":300}'
+ MANAGED_JENKINS_NODE_LABELS: '["mxnetlinux-cpu","restricted-mxnetlinux-cpu","mxnetlinux-gpu", "restricted-mxnetlinux-gpu", "mxnetwindows-cpu","mxnetwindows-gpu","mxnetlinux-gpu-p3","restricted-mxnetlinux-gpu-p3","mxnetlinux-gpu-p3-8xlarge","utility","restricted-utility"]'
+ IGNORED_JENKINS_NODE_LABELS: '["mxnetlinux","mxnetwindows","master"]'
+ IGNORED_JENKINS_NODE_NAMES: '["master"]'
+ LOGGING_LEVEL: DEBUG
+ LOGGING_LEVEL_BOTOCORE: INFO
+ LOGGING_LEVEL_BOTO3: INFO
+ LOGGING_LEVEL_URLLIB3: INFO
+ LOGGING_LEVEL_REQUESTS: ERROR
+
+prod:
+ SECRET_NAME: REDACTED
+ SECRET_ENDPOINT_URL: https://secretsmanager.us-west-2.amazonaws.com
+ SECRET_ENDPOINT_REGION: us-west-2
+ EC2_ACCOUNT_ARN: arn:aws:ec2:us-west-2:REDACTED:*
+ IAM_JENKINS_SLAVE_ROLE: arn:aws:iam::REDACTED:role/jenkins_slave_role
+ IAM_JENKINS_RESTRICTED_SLAVE_ROLE: arn:aws:iam::REDACTED:role/jenkins_restricted_slave_role
+ SECRETS_MANAGER_ARN: arn:aws:secretsmanager:us-west-2:REDACTED:secret:REDACTED
+ JENKINS_PRIV_TUNNEL: jenkins-priv.mxnet-ci.amazon-ml.com:48593
+ LAUNCH_TEMPLATES: '{"mxnetlinux-cpu":{"id":"lt-059ca0af3b73fdd43","version":"6"},"restricted-mxnetlinux-cpu":{"id":"lt-0752e01a2f18939a4","version":"4"},"mxnetlinux-gpu":{"id":"lt-083414b180618edd0","version":"8"},"mxnetlinux-gpu-p3":{"id":"lt-09e887362d145072b","version":"8"},"mxnetlinux-gpu-p3-8xlarge":{"id":"lt-03458db28b362cf92","version":"5"},"mxnetwindows-cpu":{"id":"lt-02d3dfef15faf1298","version":"6"},"mxnetwindows-gpu":{"id":"lt-0eb15ef80c9a69ef1","version":"6"},"utility":{" [...]
+ EXECUTORS_PER_LABEL: '{"mxnetlinux-cpu":3,"restricted-mxnetlinux-cpu":3,"mxnetlinux-gpu":1,"restricted-mxnetlinux-gpu":0,"mxnetlinux-gpu-p3":1,"mxnetlinux-gpu-p3-8xlarge":1,"mxnetwindows-cpu":4,"restricted-mxnetlinux-gpu-p3":1,"mxnetwindows-gpu":1,"utility":30,"restricted-utility":30}'
+ WARM_POOL_SIZE: '{"mxnetlinux-cpu":1,"restricted-mxnetlinux-cpu":0,"mxnetlinux-gpu":0,"mxnetlinux-gpu-p3":0,"mxnetlinux-gpu-p3-8xlarge":0,"mxnetwindows-cpu":1,"mxnetwindows-gpu":0,"restricted-mxnetlinux-gpu-p3":0,"utility":1,"restricted-utility":1}'
+ MINIMUM_QUEUE_TIMES_SEC: '{"mxnetlinux-cpu":30,"restricted-mxnetlinux-cpu":30,"mxnetlinux-gpu":30,"mxnetlinux-gpu-p3":30,"mxnetlinux-gpu-p3-8xlarge":30,"restricted-mxnetlinux-gpu-p3":30,"mxnetwindows-cpu":30,"mxnetwindows-gpu":30,"utility":3,"restricted-utility":3}'
+ CCACHE_EFS_DNS: '{"mxnetlinux-cpu":"fs-REDACTED.efs.us-west-2.amazonaws.com","restricted-mxnetlinux-cpu":"fs-REDACTED.efs.us-west-2.amazonaws.com","mxnetlinux-gpu":"NONE","mxnetlinux-gpu-p3":"NONE","restricted-mxnetlinux-gpu-p3":"NONE","mxnetlinux-gpu-p3-8xlarge":"NONE","mxnetwindows-cpu":"NONE","mxnetwindows-gpu":"NONE","utility":"NONE","restricted-utility":"NONE"}'
+ MAXIMUM_STARTUP_TIME_SEC: '{"mxnetlinux-cpu":300,"restricted-mxnetlinux-cpu":300,"mxnetlinux-gpu":300,"mxnetlinux-gpu-p3":300,"restricted-mxnetlinux-gpu-p3":300,"mxnetlinux-gpu-p3-8xlarge":300,"mxnetwindows-cpu":1800,"mxnetwindows-gpu":1800,"utility":300,"restricted-utility":300}'
+ MANAGED_JENKINS_NODE_LABELS: '["mxnetlinux-cpu","restricted-mxnetlinux-cpu","mxnetlinux-gpu","mxnetwindows-cpu","mxnetwindows-gpu","mxnetlinux-gpu-p3","restricted-mxnetlinux-gpu-p3","mxnetlinux-gpu-p3-8xlarge","utility","restricted-utility"]'
+ IGNORED_JENKINS_NODE_LABELS: '["mxnetlinux","mxnetwindows","master"]'
+ IGNORED_JENKINS_NODE_NAMES: '["master"]'
+ LOGGING_LEVEL: DEBUG
+ LOGGING_LEVEL_BOTOCORE: INFO
+ LOGGING_LEVEL_BOTO3: INFO
+ LOGGING_LEVEL_URLLIB3: INFO
+ LOGGING_LEVEL_REQUESTS: ERROR
diff --git a/services/jenkins-autoscaling/lambda_mxnet_ci/autoscaling/handler.py b/services/jenkins-autoscaling/lambda_mxnet_ci/autoscaling/handler.py
new file mode 100755
index 0000000..2f7da20
--- /dev/null
+++ b/services/jenkins-autoscaling/lambda_mxnet_ci/autoscaling/handler.py
@@ -0,0 +1,1533 @@
+#!/usr/bin/env python3
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# -*- coding: utf-8 -*-
+
+"""
+Handler for auto scaling of MXNets CI system. Deployed using Lambda
+"""
+
+import datetime
+import json
+import logging
+import os
+import random
+import re
+import string
+import time
+from collections import defaultdict, OrderedDict
+from urllib import parse
+from typing import DefaultDict, Dict, Any, List, Optional
+from itertools import filterfalse, tee
+
+import math
+import boto3
+from botocore.config import Config
+from botocore.exceptions import ClientError
+import jenkinsapi
+from joblib import Parallel, delayed
+
+DOWNSCALE_REASON = '[AUTOSCALING] Downscale'
+DOWNSCALE_MANUAL_REASON = '[DOWNSCALE]'
+
+# Queue reasons being output by Jenkins.
+RE_NO_AVAILABLE_NODE = [
+ r"(There are no nodes with the label ‘(?P<label>[^\s;\\]*)’)",
+ r"(All nodes of label ‘(?P<label>[^\s;\\]*)’ are offline)",
+ r"(doesn’t have label (?P<label>[^\s;\\]*))",
+ r"(Waiting for next available executor on (?P<label>[^\s;\\]*))",
+ r"((?P<label>[^\s;\\]*) is offline)",
+]
+RE_NO_AVAILABLE_NODES = r"(^Waiting for next available executor$)"
+
+# Offline cause for nodes that have been taken offline by the jenkins monitoring
+NODE_MONITOR_OFFLINE_CAUSE = 'hudson.node_monitors'
+
+# Since windows got hourly billing, we only want to consider instances which are running close to the full hour
+WINDOWS_MIN_PARTIAL_RUNTIME_SECONDS = 55 * 60
+
+# EC2s API only allows a specific number of filters. This constant defines the chunk size for these requests
+EC2_FILTER_CHUNK_SIZE = 40
+
+# Only try to scale up X number of instances per round. Otherwise, we might run into problems due to exceeding the
+# maximum execution time of lambda
+NUM_UPSCALES_PER_ROUND = 20
+
+# Only try to scale down X number of instances per round. Otherwise, we might run into problems due to exceeding the
+# maximum execution time of lambda
+NUM_DOWNSCALES_PER_ROUND = 40
+
+# Limit the number of parallel requests for AWS; otherwise, we might get our requests declined by the API
+AWS_PARALLEL_REQUESTS_LIMIT = 3
+
+# Limit the number of parallel requests for Jenkins; otherwise, we might be DOSing our own server
+JENKINS_PARALLEL_REQUESTS_LIMIT = 100
+
+# Limit the number of parallel create request for Jenkins. It tends to get into race conditions since the create
+# action is not threadsafe.....
+JENKINS_PARALLEL_CREATE_REQUESTS_LIMIT = 10
+
+# Seconds a Jenkins request might have until it times out
+JENKINS_REQUEST_TIMEOUT_SECONDS = 300
+
+
+def partition(predicate, iterable):
+ xs, ys = tee(iterable)
+ return list(filter(predicate, xs)), list(filterfalse(predicate, ys))
+
+
+def memoize(cached_function):
+ """ Memoization decorator for functions taking one or more arguments. """
+
+ # http://code.activestate.com/recipes/578231-probably-the-fastest-memoization-decorator-in-the-/
+
+ class MemoDict(dict):
+ def __init__(self, cached_function):
+ super().__init__()
+ self.cached_function = cached_function
+
+ def __call__(self, *args):
+ return self[args]
+
+ def __missing__(self, key):
+ ret = self[key] = self.cached_function(*key)
+ return ret
+
+ return MemoDict(cached_function)
+
+
+def execute_scale_up_logic(jenkins_server, ec2_resource, scale_up_nb_nodes):
+ if not scale_up_nb_nodes:
+ logging.info('No scale up required')
+ return
+
+ # Scale up: Create jenkins node slots
+ _scale_up_slots = _create_jenkins_node_slots(jenkins_server=jenkins_server, label2num_instances=scale_up_nb_nodes)
+
+ # Scale up: Start nodes
+ started_instance_names = _launch_ec2_instances(scale_up_slots=_scale_up_slots, ec2_resource=ec2_resource)
+ not_started_instance_names = set([name for label, name_list in _scale_up_slots.items() for name in name_list]). \
+ difference(set(started_instance_names))
+
+ # Delete jenkins slave node entries which have not been started
+ if not_started_instance_names:
+ logging.warning('The following instances have not been started: %s', ', '.join(not_started_instance_names))
+ not_started_node_objs = _convert_to_jenkins_nodes(server=jenkins_server, instances=not_started_instance_names)
+ _delete_jenkins_node_objects(node_objs=not_started_node_objs)
+
+
+def execute_scale_down_logic(jenkins_server, ec2_resource, scale_down_nodes):
+ if not scale_down_nodes:
+ logging.info('No scale down required')
+ return
+
+ # Extract all display names from scale_down_nodes. Node is an api internal dictionary,
+ # coming from jenkinsapi.computers._data
+ scale_down_node_names = [node['displayName'] for nodes in scale_down_nodes.values() for node in nodes]
+ scale_down_nodes_list = list(_convert_to_jenkins_nodes(jenkins_server, scale_down_node_names).values())
+
+ (nodes_online, nodes_offline) = partition(lambda node: not node._data['offline'], scale_down_nodes_list)
+
+ # Mark nodes as offline to prevent builds being scheduled
+ _mark_nodes_offline(offline_nodes=nodes_online, reason=DOWNSCALE_REASON)
+
+ # Nodes that are offline but executing jobs are marked online again
+ # Scale down: Validate nodes are properly taken offline and no builds are going to be interrupted
+ final_scale_down_nodes_list, non_idle_offline_nodes = _partition_non_idle(nodes_online)
+ # Re-enable filtered nodes
+ logging.debug('Re-enabling %d non-idle nodes: %s', len(non_idle_offline_nodes), non_idle_offline_nodes)
+ _mark_nodes_online(online_nodes=non_idle_offline_nodes)
+
+ final_scale_down_nodes_list.extend(nodes_offline)
+
+ # Scale down: Shutdown node instances
+ shutdown_ec2_instance_name_list = [node.name for node in final_scale_down_nodes_list]
+ logging.debug('Shutting down %d instances: %s', len(shutdown_ec2_instance_name_list),
+ shutdown_ec2_instance_name_list)
+
+ #########################################################
+ _terminate_ec2_instances(
+ instance_names=shutdown_ec2_instance_name_list,
+ ec2_resource=ec2_resource)
+ #########################################################
+
+ terminated_nodes = _convert_to_jenkins_nodes(
+ server=jenkins_server,
+ instances=shutdown_ec2_instance_name_list)
+
+ logging.debug('Deleting %d jenkins nodes: %s', len(terminated_nodes), terminated_nodes)
+ #########################################################
+ _delete_jenkins_node_objects(node_objs=terminated_nodes)
+ #########################################################
+
+
+def determine_scale_up_nodes(queue_items: List[Dict[str, Any]], nodes: List[Dict[str, Any]],
+ unconnected: Dict[str, List[str]]) -> Dict[str, int]:
+ """
+ Determine the number of nodes which are required of each label
+ :param queue_items: Currently enqueued items
+ :param nodes: Currently connected nodes
+ :param unconnected: instances which are starting or not yet connected
+ :return: Dict(label, nb_required_nodes)
+ """
+ dict_required_executors: DefaultDict[str, int] = defaultdict(int)
+
+ idle_nodes_per_label = _get_idle_nodes_per_label(nodes_data=nodes)
+
+ cur_time_s = time.time()
+ for queue_item in queue_items:
+ # Make sure we're only processing queue items that are related to resource starvation
+ label = _label_from_queued_job(nodes=nodes, queue_item=queue_item)
+ if label:
+ if label not in _minimum_queue_times_sec(): # pragma: no cover
+ logging.error("Label %s from queue reason '%s' is not part of MINIMUM_QUEUE_TIMES_SEC - skipping..",
+ label, queue_item['why'])
+ continue
+
+ # Only consider items which have been in the queue for a specific time. This ensure we're not scaling
+ # too aggressively.
+ queue_duration_s = cur_time_s - (queue_item['inQueueSince'] / 1000)
+ if queue_duration_s < _minimum_queue_times_sec()[label]:
+ logging.debug('Queue duration of item %s is not mature enough: %d<%d',
+ queue_item['id'], queue_duration_s, _minimum_queue_times_sec()[label])
+ continue
+
+ # See if there are actually no nodes available or if the problem is actually that this job has no permission
+ # to run on a restricted node. Unfortunately, we can't access the individual executors, so we have to rely
+ # on the is_idle metric. TODO: Find out if we can get more detailed data or maybe even blacklist a job
+ # Unfortunately, this will trigger a ping-pong in scale up and down since we're unable to determine ahead of
+ # time whether a job is actually lacking nodes to run on or whether it's just ineligible. For now, we will
+ # throw an error. In future, this could automatically be handled by another job. Also, an investigation
+ # should be kicked off since this could mean somebody is trying to run on a restricted slave without
+ # permission.
+ if label in idle_nodes_per_label and idle_nodes_per_label[label] > 0:
+ logging.error('Queue item %s is scheduled for label %s, but there are %d idle nodes available. This is '
+ 'most likely somebody unauthorized trying to schedule an unrestricted job onto a '
+ 'restricted slave. Please investigate by checking the job queue.',
+ queue_item['id'], label, idle_nodes_per_label[label])
+ continue
+
+ dict_required_executors[label] = dict_required_executors.get(label, 0) + 1
+ else:
+ logging.debug('Queue item %s is not related to resource starvation: %s',
+ queue_item['id'], queue_item['why'])
+
+ label2num_instances = _calculate_nb_required_nodes(dict_required_executors=dict_required_executors)
+
+ # substract the number of unconnected instances
+ for label, names in unconnected.items():
+ logging.debug('%d nodes of type %s currently starting', len(names), label)
+ resulting_number = label2num_instances.get(label, 0) - len(names)
+ resulting_number = max(0, resulting_number) # Take negative numbers into account
+ if resulting_number > 0:
+ label2num_instances[label] = resulting_number
+ logging.debug('%d new nodes for %s required (down from %d)', resulting_number, label, len(names))
+ else:
+ if label in label2num_instances:
+ label2num_instances.pop(label)
+ logging.debug('No new nodes for %s required (down from %d)', label, len(names))
+
+ return label2num_instances
+
+
+def determine_scale_down_nodes(nodes_data: List[Dict[str, Any]], instance_uptime: Dict[str, int]) \
+ -> Dict[str, List[str]]:
+ """
+ Determine which instances should be shut down due to idle
+ :param nodes_data: Currently connected nodes (List of dicts)
+ :param instance_uptime: Duration about how long each node has been running
+ :return: Dict(label, list(nodes_to_disable))
+ """
+ nodes_to_disable: DefaultDict[str, list] = defaultdict(list)
+ considered_nodes: DefaultDict[str, list] = defaultdict(list)
+
+ for node_data in nodes_data:
+ if not node_data['offline'] and node_data['idle']:
+ display_name = node_data['displayName']
+ label = _managed_node_label(node_data)
+ if not label:
+ logging.error('Could not extract the managed label for node %s', display_name)
+ continue
+
+ # Check if label is managed - otherwise skip
+ if label not in _managed_jenkins_node_labels():
+ logging.debug('Label %s is not managed, skipping...', label)
+ continue
+
+ # TODO: Add a label that marks reserved instances
+
+ if node_data['monitorData']['hudson.node_monitors.ArchitectureMonitor'] is None:
+ # Sometimes, the architecture monitor is not set. This is a race condition and can be
+ # ignored since the information is available within the next turn
+ logging.info('Architecture has not been propagated for %s, ignoring until next scale_down check',
+ display_name)
+ continue
+
+ # Windows instances are getting billed hourly and need special handling
+ if 'Windows' in node_data['monitorData']['hudson.node_monitors.ArchitectureMonitor']:
+ if display_name not in instance_uptime:
+ logging.error('Unable to find uptime for %s', display_name)
+ continue
+
+ running_duration_seconds = instance_uptime[display_name]
+ running_duration_partial = running_duration_seconds % (60 * 60)
+ # Don't shutdown instances below XXh50min uptime to make use of hourly billing
+ if running_duration_partial < WINDOWS_MIN_PARTIAL_RUNTIME_SECONDS:
+ considered_nodes[label].append(node_data)
+ logging.debug(
+ 'Ignoring %s because partial runtime %ds is below limit of %ds (hourly billing). Total '
+ 'runtime: %ds',
+ display_name, running_duration_partial, WINDOWS_MIN_PARTIAL_RUNTIME_SECONDS,
+ running_duration_seconds)
+ continue
+
+ # TODO: Check for how long an instance has been idling. There is no built-in API for now and the
+ # only way is to go through the entire Jenkins build history. Save this up for later.
+
+ nodes_to_disable[label].append(node_data)
+ considered_nodes[label].append(node_data)
+
+ # Leave some buffer for warm pool. This code makes sure to always leave X instances in idle while scaling down.
+ # For example: 5 instances running, 3 in idle, WARM_POOL_SIZE set to 2. This code will remove only 1 instance,
+ # leading to 4 instances running and 2 in idle.
+ for warm_label, warm_value in _warm_pool_node_counts().items():
+ cur_nodes = nodes_to_disable[warm_label]
+ cur_considered_nodes = considered_nodes[warm_label]
+ if cur_nodes:
+ warm_value -= (len(cur_considered_nodes) - len(cur_nodes))
+ for _ in range(0, warm_value):
+ # Pop a random entry. Otherwise, the first node is never going to be shut down
+ cur_nodes.pop(random.randrange(0, len(cur_nodes)))
+
+ # Remove empty lists, caused by the usage of defaultdict()
+ return {key: val for key, val in nodes_to_disable.items() if val}
+
+
+def _determine_faulty_nodes(nodes: List[Dict[str, Any]], unconnected_instances: Dict[str, List[str]],
+ instance_uptime: Dict[str, int]) -> Dict[str, List[Any]]:
+ """
+ Determine all nodes that are in a faulty state and should thus be turned off
+ :param unconnected_instances: Names of all nodes that are currently starting up
+ :param nodes: Currently connected nodes (List of dicts)
+ :param instance_uptime: Duration about how long each node has been running
+ :return: (Dict[label, List(node)] containing nodes that are faulty, instances not found in jenkins)
+
+ """
+ label2faulty_nodes: DefaultDict[str, List[Any]] = defaultdict(list)
+ orphaned_instances = []
+
+ # Determine instances that failed to start up. This sometimes happens with windows slaves
+ for label, instances in unconnected_instances.items():
+ maximum_startup_limit = _maximum_startup_time()[label]
+ for instance in instances:
+ node = _find_node_by_name(nodes, instance)
+ if not node:
+ # Autoscaling instance not known to jenkins
+ logging.error('Could not find node_data for %s, marked as orphaned instance for termination', instance)
+ orphaned_instances.append(instance)
+ continue
+
+ uptime = instance_uptime[instance]
+ if uptime > maximum_startup_limit:
+ logging.warning('Instance %s failed to start up within %d seconds', instance, uptime)
+ label2faulty_nodes[label].append(node)
+
+ for node in nodes:
+ if node['displayName'] == 'master':
+ # Don't do anything for master
+ continue
+
+ label = _managed_node_label(node)
+
+ if not label:
+ logging.info('Slave %s is not managed by auto scaling. Ignoring for consideration of faulty instances',
+ node['displayName'])
+ # Turn off slaves that have been marked as offline by Jenkins due to things like too low disk space
+ elif node['temporarilyOffline'] and node['offlineCause'] and \
+ node['offlineCause']['_class'].startswith(NODE_MONITOR_OFFLINE_CAUSE):
+ logging.warning('Instance %s has been marked as offline by Jenkins monitoring due to "%s"',
+ node['displayName'], node['offlineCauseReason'])
+ label2faulty_nodes[label].append(node)
+ # Turn off slaves that have been marked to downscale but have not been downscaled
+ elif node['offlineCauseReason'] == DOWNSCALE_REASON or node['offlineCauseReason'] \
+ .startswith(DOWNSCALE_MANUAL_REASON):
+ logging.warning('Instance %s has been marked to downscale but has not scaled down: "%s"',
+ node['displayName'], node['offlineCauseReason'])
+ label2faulty_nodes[label].append(node)
+ # Delete node slots that have been created but dont have a matching instance
+ elif node['displayName'] not in instance_uptime:
+ logging.warning('Slot for %s has been created but instance never started', node['displayName'])
+ label2faulty_nodes[label].append(node)
+
+ # Remove empty lists, caused by the usage of defaultdict()
+ return ({key: val for key, val in label2faulty_nodes.items() if val}, orphaned_instances)
+
+
+def _calculate_nb_required_nodes(dict_required_executors):
+ """
+ Calculate the number of required nodes based on the number of required executors for each label
+ :param dict_required_executors: Dict[label, nb_required_executors]
+ :return: Dict[label, nb_required_nodes]
+ """
+ dict_required_nodes = dict()
+ for label, nb_required_executors in dict_required_executors.items():
+ # Old jobs are still being triggered by some people, don't throw an error in that case
+ if label in _ignored_jenkins_node_labels():
+ logging.debug('Skipping ignored label %s', label)
+ continue
+
+ # Search for node with that label to extract how many executors it has
+ nb_executors_per_node = _get_nb_executors_by_label(label)
+ if nb_executors_per_node and nb_executors_per_node > 0:
+ nb_required_nodes = math.ceil(nb_required_executors / nb_executors_per_node)
+ logging.info('Need %d nodes for %d executors of type %s',
+ nb_required_nodes, nb_required_executors, label)
+
+ if nb_required_nodes > 0:
+ dict_required_nodes[label] = nb_required_nodes
+ else:
+ logging.error('Node label %s has %s executors per node. Has to be positive.',
+ label, nb_executors_per_node)
+
+ return dict_required_nodes
+
+
+def _unconnected_instances(nodes: list, instance_uptime: Dict[str, int], ec2_resource) \
+ -> Dict[str, List[str]]:
+ """
+ Returns instances which are currently starting up but not connected yet
+ :return: Dict, mapping label to List of slave names that are still starting up
+ """
+ dict_starting_nodes: Dict[str, List[str]] = defaultdict(list)
+ instances_filter = ec2_resource.instances.filter(
+ Filters=[
+ {'Name': 'instance-state-name', 'Values': ['starting', 'running']},
+ {'Name': 'tag:AutoScaledSlave', 'Values': ['True']} # Ensure only listing instances managed by auto scaling
+ ])
+ for instance in instances_filter:
+ tags = _ec2Instance_tag_dict(instance)
+ target_node = _find_node_by_name(nodes=nodes, name=tags['Name'])
+ if target_node and target_node['offline'] and (not target_node['temporarilyOffline']):
+ logging.debug('Instance %s starting up but not connected yet', target_node['displayName'])
+ if 'label' in tags:
+ label = tags['label']
+ uptime_seconds = instance_uptime[target_node['displayName']]
+ logging.info('Instance %s - %s of type %s is starting up for %d seconds.',
+ instance.id, tags['Name'], label, uptime_seconds)
+
+ dict_starting_nodes[label].append(tags['Name'])
+ else: # pragma: no cover
+ logging.error("Managed slave instance %s does not have tag label", instance.id)
+
+ return dict_starting_nodes
+
+
+def _label_from_queued_job(nodes, queue_item) -> Optional[str]:
+ """
+ Extract the node type label from a queue item. The queue item contains a reason that states why it's currently
+ hanging and exposes the name of the label it's waiting for. This label is extracted by this method.
+ :param queue_item: Queue item dict
+ :return: Label
+ """
+ # Check if there are no running nodes in general. This is a special case since jenkins does not tell which
+ # nodes are actually required. In that case, just assume we need a ubuntu-cpu executor.
+ regex_result = re.search(RE_NO_AVAILABLE_NODES, queue_item['why'])
+ if regex_result:
+ logging.debug('There are no nodes on the Jenkins master, creating mxnetlinux-cpu nodes to start'
+ ' label propagation')
+ label = 'mxnetlinux-cpu'
+ else:
+ for re_available_node in RE_NO_AVAILABLE_NODE:
+ regex_result = re.search(re_available_node, queue_item['why'])
+ if regex_result:
+ label = regex_result.group('label')
+ break
+ else:
+ return None
+
+ # Clean up label of any other characters
+ label = label.strip(' ‘’\'"')
+
+ # Sometimes, Jenkins does not put the required label into the message but a node-name instead.
+ # In this case, we have to extract the label from the node
+ if label not in _managed_jenkins_node_labels():
+ node = _find_node_by_name(nodes=nodes, name=label)
+ if not node:
+ logging.error("Queue reason '%s' contains unresolvable label '%s'", queue_item['why'], label)
+ return None
+ label = _managed_node_label(node=node)
+ if not label:
+ logging.error('Could not extract type label for node %s as part of queue reason "%s"',
+ node['displayName'], queue_item['why'])
+ return None
+ return label
+
+
+def _get_nb_executors_by_label(label) -> Optional[int]:
+ """
+ Return the number of executors that can be run on
+ :param label: Node label
+ :return: Number of executors available for that label
+ """
+ if label in _get_nb_executors_per_label():
+ return _get_nb_executors_per_label()[label]
+ logging.error('Label %s is not part of NB_EXECUTORS_PER_LABEL', label)
+ return None
+
+
+def _find_node_by_name(nodes, name):
+ """
+ Loop through a list of nodes and return the one which has the matching display name
+ :param nodes: List of nodes
+ :param name: Target name
+ :return: Matching node, None otherwise
+ """
+ # Nodes always have unique names, thus there's no need for duplicate check
+ for node in nodes:
+ if name == node['displayName']:
+ return node
+ return None
+
+
+def _mark_nodes_offline(offline_nodes, reason):
+ """
+ Mark jenkins nodes as offline
+ :param server: Jenkins server handle
+ :param offline_nodes: List of nodes to mark as offline
+ :return: None
+ """
+ if not offline_nodes:
+ logging.info('No nodes to be marked as offline')
+ else:
+ logging.info('Marking %s as offline', [x.name for x in offline_nodes])
+ Parallel(n_jobs=min(JENKINS_PARALLEL_REQUESTS_LIMIT, len(offline_nodes)), backend="threading")(
+ delayed(node_obj.set_offline)(reason) for node_obj in offline_nodes)
+
+
+def _mark_nodes_online(online_nodes):
+ """
+ Mark jenkins nodes as online
+ :param server: Jenkins server handle
+ :param offline_nodes: List of nodes to mark as online
+ :return: None
+ """
+ if not online_nodes:
+ logging.debug('No nodes to be marked as online')
+ else:
+ logging.info('Marking %s as online', online_nodes)
+ Parallel(n_jobs=min(JENKINS_PARALLEL_REQUESTS_LIMIT, len(online_nodes)), backend="threading")(
+ delayed(node_obj.set_online)() for node_obj in online_nodes)
+
+
+def _convert_to_jenkins_nodes(server, instances):
+ """
+ Take a list of jenkins node names and convert them to API backed Node objects
+ :param server: Jenkins server handle
+ :param instances: Node names
+ :return: Dict(displayName, node_obj)
+ """
+ nodes = dict()
+
+ if not instances:
+ return nodes
+
+ # Unfortunately, the iterator of the jenkinsapi always requests the data of every single node upon calling the
+ # dict (e.g. all_nodes[nodename]).
+ # Since this is basically O(MN); M=len(offline_node_names); N=number of total nodes, we have to use a workaround
+ # and create the objects ourselves instead of using the underlying dict - which is getting created in sequence...
+ node_obj_list = Parallel(n_jobs=min(JENKINS_PARALLEL_REQUESTS_LIMIT, len(instances)), backend="threading")(
+ delayed(_create_jenkins_node_obj)(server, x) for x in instances)
+
+ for node_obj in node_obj_list:
+ if node_obj:
+ nodes[node_obj.name] = node_obj
+ return nodes
+
+
+# This has to be defined at this scope because joblib is not able to access functions defined in the local scope
+def _create_jenkins_node_obj(server, node_name):
+ """
+ Helper function to create a jenkins node object.
+ :return: Jenkins node object
+ """
+ try:
+ node = jenkinsapi.node.Node(jenkins_obj=server, baseurl='{}/computer/{}'.format(server.baseurl, node_name),
+ nodename=node_name, node_dict={})
+ return node
+ except Exception as e:
+ logging.error("_create_jenkins_node_obj:")
+ logging.exception(e)
+ return None
+
+
+def _partition_non_idle(node_objs):
+ """
+ Return nodes marked to be taken offline but still running a job. By design, nodes marked as offline are able to
+ finish their scheduled jobs and don't get interrupted. This function filters nodes that got a job scheduled.
+ :param node_objs: Jenkins nodes
+ :return: 1. List of node_obj that are marked as offline and idling. 2. List of non-idle nodes
+ """
+ if not node_objs:
+ return [], []
+
+ # Update all objects before checking their content
+ Parallel(n_jobs=min(JENKINS_PARALLEL_REQUESTS_LIMIT, len(node_objs)), backend="threading")(
+ delayed(node_obj.poll)() for node_obj in node_objs)
+ logging.debug("%d jenkins nodes updated", len(node_objs))
+
+ new_node_objs = [node_obj for node_obj in node_objs if ((node_obj.is_idle()) and not node_obj.is_online())]
+ non_idle_node_objs = [node_obj for node_obj in node_objs if ((not node_obj.is_idle()) or node_obj.is_online())]
+ assert len(new_node_objs) + len(non_idle_node_objs) == len(node_objs)
+
+ if non_idle_node_objs:
+ logging.info('%s got a job scheduled while they are marked as offline - possible race condition',
+ ', '.join(str(o) for o in non_idle_node_objs))
+
+ return new_node_objs, non_idle_node_objs
+
+
+def _instance_uptime(ec2_resource) -> Dict[str, int]:
+ """
+ Return how long each instance has been running so far (uptime measured as time since launch, not OS uptime)
+ :return: Dict[instance-name : uptime in seconds]
+ """
+
+ instances = list(ec2_resource.instances.filter(
+ Filters=[
+ {'Name': 'tag:AutoScaledSlave', 'Values': ['True']} # Ensure only listing instances managed by auto scaling
+ , {'Name': 'instance-state-name', 'Values': ['starting', 'running']}
+ ]
+ ))
+
+ valid_instances = filter_ignored(instances)
+ instance_uptime = {}
+ if valid_instances:
+ current_datetime = datetime.datetime.now(valid_instances[0].launch_time.tzinfo)
+ for instance in valid_instances:
+ name = _ec2Instance_tag_dict(instance)['Name']
+ duration = (current_datetime - instance.launch_time).total_seconds()
+ instance_uptime[name] = duration
+
+ return instance_uptime
+
+
+def _terminate_ec2_instances(instance_names, ec2_resource):
+ """
+ Shutdown the instances which the nodes with the passed names are running on
+ :param instance_names: List of node names that should be shut down
+ :param ec2_resource: EC2 resource handle
+ """
+ if instance_names:
+ logging.info("Terminating %d instances: %s", len(instance_names), instance_names)
+
+ # Prevent botocore.exceptions.ClientError: An error occurred (FilterLimitExceeded) when calling the
+ # DescribeInstances operation: The maximum number of filter values specified on a single call is 200
+ instance_names_chunked = chunks(source_list=instance_names, chunk_size=EC2_FILTER_CHUNK_SIZE)
+ for instance_names in instance_names_chunked:
+ ec2_resource.instances.filter(
+ Filters=
+ [
+ {'Name': 'tag:Name', 'Values': instance_names},
+ {'Name': 'tag:AutoScaledSlave', 'Values': ['True']}
+ # Ensure only listing instances managed by auto scaling
+ ]
+ ).terminate()
+
+
+def filter_ignored(ec2_instances):
+ ignored = set(_ignored_jenkins_node_names())
+ return list(filter(lambda x: _ec2Instance_tag_dict(x)['Name'] not in ignored, ec2_instances))
+
+
+def _delete_jenkins_node_objects(node_objs):
+ """
+ Delete the jenkins slave node entries
+ :param node_names: List of node objs
+ """
+ if not node_objs:
+ logging.debug('No jenkins nodes to delete, exiting eagerly')
+ else:
+ Parallel(n_jobs=min(JENKINS_PARALLEL_REQUESTS_LIMIT, len(node_objs)), backend="threading")(
+ delayed(_delete_jenkins_node_object)(node_obj) for node_obj in node_objs.values())
+
+
+def _delete_jenkins_node_object(node_obj):
+ """
+ Delete a jenkins node object from the jenkins master
+ :param node_obj:
+ :return:
+ """
+ from requests.exceptions import HTTPError
+ try:
+ if node_obj.is_online():
+ logging.error('Unable to delete still connected jenkins node %s', node_obj.name)
+ return
+
+ delete_url = "{}/doDelete".format(node_obj.baseurl)
+ node_obj.jenkins.requester.post_and_confirm_status(url=delete_url, data={}, allow_redirects=False)
+ except HTTPError as e:
+ # The JenkinsAPI returns a 404 if a slave does not exist. Since we are trying to delete it in this call,
+ # that's totally fine. The background is the concurrency and inconsistency of the Jenkins backend. Sometimes,
+ # a slave is still being returned as existent although it already has been deleted. We can gracefully ignore
+ # that exception then.
+ if e.response.status_code == 404:
+ logging.debug('Slave has been deleted already, no need to delete %s', node_obj.baseurl)
+ else:
+ raise
+
+
+def _launch_ec2_instances(scale_up_slots, ec2_resource):
+ """
+ Launch ec2 instances, matching the appropriate labels
+ :param scale_up_slots:
+ :param ec2_resource:
+ :return: List of started instance names. Allows to determine whether some instances were not started.
+ """
+ jobs = []
+ launch_templates = _launch_templates()
+
+ # Start each instance one by one as each of them require different user data
+ for label, target_instance_names in scale_up_slots.items():
+ if label not in launch_templates:
+ logging.error('No launch template for %s defined', label)
+ continue
+
+ launch_template = launch_templates[label]
+ launch_template_id = launch_template['id']
+ launch_template_version = launch_template['version']
+
+ for target_instance_name in target_instance_names:
+ logging.debug('Launching instance %s of type %s', target_instance_name, label)
+ user_data_command = _format_ec2_user_data_command(label=label, target_instance_name=target_instance_name)
+ if user_data_command is None: # pragma: no cover
+ logging.error('No user data command defined for %s, skipping...', target_instance_name)
+ continue
+ else:
+ # Enqueue job
+ jobs.append({
+ 'label': label,
+ 'target_instance_name': target_instance_name,
+ 'launch_template_id': launch_template_id,
+ 'launch_template_version': launch_template_version,
+ 'user_data_command': user_data_command
+ })
+
+ started_instance_names = Parallel(n_jobs=min(AWS_PARALLEL_REQUESTS_LIMIT, len(jobs)), backend="threading")(
+ delayed(_launch_ec2_instance)(ec2_resource=ec2_resource, label=job['label'],
+ target_instance_name=job['target_instance_name'],
+ launch_template_id=job['launch_template_id'],
+ launch_template_version=job['launch_template_version'],
+ user_data_command=job['user_data_command']) for job in jobs)
+
+ return [x for x in started_instance_names if x is not None]
+
+
+def _launch_ec2_instance(ec2_resource, label, target_instance_name, launch_template_id, launch_template_version,
+ user_data_command):
+ try:
+ ec2_resource.meta.client.run_instances(
+ DryRun=False,
+ MaxCount=1,
+ MinCount=1,
+ LaunchTemplate={
+ 'LaunchTemplateId': launch_template_id,
+ 'Version': launch_template_version
+ },
+ TagSpecifications=[
+ {
+ 'ResourceType': 'instance',
+ 'Tags': [
+ {
+ 'Key': 'Name',
+ 'Value': target_instance_name
+ },
+ {
+ 'Key': 'AutoScaledSlave',
+ 'Value': 'True'
+ },
+ {
+ 'Key': 'label',
+ 'Value': label
+ }
+ ]
+ },
+ ],
+ UserData=user_data_command
+ )
+ return target_instance_name
+ except ClientError as client_error:
+ error_code = client_error.response['Error']['Code']
+ if error_code == 'InsufficientInstanceCapacity':
+ logging.info("Insufficient instance capacity, can't launch %s: %s",
+ target_instance_name, client_error.response['Error']['Message'])
+ else:
+ logging.exception('Exception occurred during instance launch')
+
+ # Make sure to catch everything, otherwise we won't be able to clean up properly
+ except Exception: # pylint: disable=broad-except
+ logging.exception('Unexpected exception during instance launch')
+
+
+def _format_ec2_user_data_command(label, target_instance_name):
+ """
+ Format the EC2 user data command that is being passed to a started instance
+ :return:
+ """
+ jenkins_credentials = _get_jenkins_credentials()
+
+ def format_windows(target_instance_name):
+ # Make sure this has no indentation or it will be invalid powershell code!
+ return \
+ """\
+<script>
+mkdir C:\jenkins_slave
+cd C:\jenkins_slave
+@echo {JENKINS_PUBLIC_URL}> jenkins_master_url.txt
+@echo {JENKINS_PRIVATE_URL}> jenkins_master_private_url.txt
+@echo {SLAVE_NAME}> jenkins_slave_name.txt
+</script>
+ """.format(
+ JENKINS_PUBLIC_URL=jenkins_credentials['jenkins_url'],
+ JENKINS_PRIVATE_URL=jenkins_credentials['jenkins_priv_url'],
+ SLAVE_NAME=target_instance_name
+ )
+
+ def format_linux(label, target_instance_name):
+ # Make sure this has no indentation or it will be invalid bash code!
+ return \
+ """\
+#!/bin/bash
+echo '{JENKINS_PUBLIC_URL}' > /home/jenkins_slave/jenkins_master_url
+echo '{JENKINS_PRIVATE_URL}' > /home/jenkins_slave/jenkins_master_private_url
+echo '{SLAVE_NAME}' > /home/jenkins_slave/jenkins_slave_name
+echo '{EFS_DNS_ADDRESS}' > /home/jenkins_slave/ccache_efs_address
+
+ """.format(
+ JENKINS_PUBLIC_URL=jenkins_credentials['jenkins_url'],
+ JENKINS_PRIVATE_URL=jenkins_credentials['jenkins_priv_url'],
+ SLAVE_NAME=target_instance_name,
+ EFS_DNS_ADDRESS=_ccache_efs_addresses()[label]
+ )
+
+ linux_types = ['mxnetlinux-cpu',
+ 'restricted-mxnetlinux-cpu',
+ 'mxnetlinux-gpu',
+ 'mxnetlinux-gpu-p3',
+ 'restricted-mxnetlinux-gpu-p3',
+ 'restricted-mxnetlinux-gpu',
+ 'mxnetlinux-gpu-p3-8xlarge',
+ 'utility',
+ 'restricted-utility']
+
+ windows_types = ['mxnetwindows-cpu',
+ 'mxnetwindows-gpu']
+
+ if label in linux_types:
+ return format_linux(label=label, target_instance_name=target_instance_name)
+ if label in windows_types:
+ return format_windows(target_instance_name=target_instance_name)
+
+ logging.error('Unable to find user data handler for %s', label)
+ return None
+
+
+def _create_jenkins_node_slots(jenkins_server, label2num_instances):
+ """
+ Create a
+ :param label:
+ :return:
+ """
+ # We could use the swarm plugin to do this automatically, but we don't want to deploy credentials to our slaves.
+ # Thus, we create nodes ourselves.
+ jobs = []
+ node_slots = defaultdict(list)
+ for label, num_instances in label2num_instances.items():
+ logging.info('Creating %d nodes of type %s', num_instances, label)
+
+ if label not in _get_slave_configuration():
+ logging.error('No slave configuration for %s found', label)
+ continue
+ configuration = _get_slave_configuration()[label]
+
+ for _ in range(0, num_instances):
+ random_part = ''.join(random.choices(string.ascii_lowercase + string.digits, k=10))
+ name = '{}_{}'.format(label, random_part)
+ logging.debug('Creating slave slot %s of type %s', name, label)
+ # Create node slot through Jenkins API. Unfortunately, we have to write our own implementation of
+ # node.Node.get_node_attributes() to support all required options. This also reimplements
+ # jenkins_server.nodes.create_node due to inefficiency of underlying implementation
+ node_attributes_encoded = parse.urlencode(
+ _custom_get_node_attributes(name=name, node_attributes=configuration))
+ url = ('{}/computer/doCreateItem?{}'.format(jenkins_server.baseurl,
+ node_attributes_encoded))
+ data = {'json': node_attributes_encoded}
+ jobs.append({
+ 'url': url,
+ 'data': data
+ })
+
+ node_slots[label].append(name)
+
+ if jobs:
+ Parallel(n_jobs=min(JENKINS_PARALLEL_CREATE_REQUESTS_LIMIT, len(jobs)), backend="threading")(
+ delayed(jenkins_server.requester.post_and_confirm_status)(url=job['url'], data=job['data'],
+ allow_redirects=False) for job in jobs)
+ else:
+ logging.debug('No jenkins node slot to create')
+
+ return node_slots
+
+
+def _custom_get_node_attributes(name, node_attributes):
+ """
+ Custom implementation of node.Node.get_node_attributes() in order to specify more options.
+
+ :return: Node attributes dict formatted for Jenkins API request
+ to create node
+ """
+
+ launcher = {
+ # http://javadoc.jenkins.io/archive/jenkins-2.73/index.html?hudson/slaves/JNLPLauncher.html
+ 'stapler-class': 'hudson.slaves.JNLPLauncher',
+ 'tunnel': node_attributes['tunnel'] # Custom option
+ }
+
+ retention = {
+ 'stapler-class': 'hudson.slaves.RetentionStrategy$Always',
+ '$class': 'hudson.slaves.RetentionStrategy$Always'
+ }
+
+ node_props = {
+ 'stapler-class-bag': 'true'
+ }
+
+ if node_attributes['job_name_restriction_regex']:
+ node_props['com.synopsys.arc.jenkinsci.plugins.jobrestrictions.nodes.JobRestrictionProperty'] = {
+ '$plugin': 'job-restrictions@0.7',
+ 'jobRestriction': {
+ 'stapler-class':
+ 'com.synopsys.arc.jenkinsci.plugins.jobrestrictions.restrictions.job.RegexNameRestriction',
+ '$class': 'com.synopsys.arc.jenkinsci.plugins.jobrestrictions.restrictions.job.RegexNameRestriction',
+ 'regexExpression': node_attributes['job_name_restriction_regex'],
+ 'checkShortName': 'false'
+ }
+ }
+
+ params = {
+ 'name': name,
+ 'type': 'hudson.slaves.DumbSlave$DescriptorImpl',
+ 'json': json.dumps({
+ 'name': name,
+ 'nodeDescription': node_attributes['node_description'],
+ 'numExecutors': node_attributes['num_executors'],
+ 'remoteFS': node_attributes['remote_fs'],
+ 'labelString': node_attributes['labels'],
+ 'mode': 'EXCLUSIVE' if node_attributes['exclusive'] else 'NORMAL',
+ 'retentionStrategy': retention,
+ 'type': 'hudson.slaves.DumbSlave',
+ 'nodeProperties': node_props,
+ 'launcher': launcher
+ })
+ }
+
+ return params
+
+
+def _get_idle_nodes_per_label(nodes_data: List[Dict[str, Any]]) -> Dict[str, int]:
+ """
+ Determine how many nodes of each label are in idle
+ :param nodes_data: Jenkins nodes data
+ :return: Dict[label, count]
+ """
+ results: DefaultDict[str, int] = defaultdict(int)
+ for node_data in nodes_data:
+ if node_data['offline'] or not node_data['idle']:
+ # Node offline or active, ignore it
+ continue
+
+ label = _managed_node_label(node_data)
+ results[label] = results.get(label, 0) + 1
+
+ return results
+
+
+def _managed_node_label(node) -> str:
+ """
+ Extract the node label e.g. mxnet-linux-cpu or mxnet-windows-gpu from a jenkins node item.
+ Nodes have several labels: such as mxnetlinux-cpu and mxnetlinux-cpu_<random tag> we return the managed one.
+ :param node: Node
+ :return: string
+ """
+ display_name = node['displayName']
+ assigned_labels = set()
+ # Convert list of tuples ('name' : 'label') to set
+ for labels_dict in node['assignedLabels']:
+ assigned_labels.add(labels_dict['name'])
+
+ # Check if blacklisted
+ intersection_blacklist = assigned_labels.intersection(_ignored_jenkins_node_labels())
+ if intersection_blacklist:
+ logging.debug('Node %s matches blacklisted labels (%s)',
+ display_name, ' & '.join(intersection_blacklist))
+ return next(iter(intersection_blacklist))
+
+ intersection_match = assigned_labels.intersection(_managed_jenkins_node_labels())
+ if len(intersection_match) > 1:
+ logging.error('Node %s has %d matching managed labels: (%s)',
+ display_name, len(intersection_match), ' & '.join(intersection_match))
+ return None
+
+ if not intersection_match:
+ logging.warning('Node %s has no matching managed labels. Assigned labels: (%s)',
+ display_name, ' & '.join(assigned_labels))
+ return None
+
+ # Found one matching label - that's the one we're looking for
+ return next(iter(intersection_match))
+
+
+def _apply_upscale_limit(limit, label2num_instances):
+ """
+ Cut-off numbers if the sum is above the limits
+ :param limit: Maximum number of instances
+ :param label2num_instances: Requested instances
+ :return: Filtered dict of instances
+ """
+ # Ordering the dict ascendingly (1 -> 2 -> 5 -> 7 -> etc) allows to ensure the floating point imprecision
+ # rather hits the biggest value and cuts that one off by one or two rather than cutting off the instances
+ # which are less required. This is especially significant in cases where only 1 instance was requested
+ # which would've resulted in the value being reduced to 0.
+ ordered_label2num_instances = OrderedDict(sorted(label2num_instances.items(), key=lambda t: t[1]))
+ new_scale_up_num_nodes = dict()
+ total_num_requested = sum(label2num_instances.values())
+ if total_num_requested <= limit:
+ # No cut-off needed
+ return label2num_instances
+
+ reduction_factor = limit / total_num_requested
+ num_nodes_left = limit
+ for label, num_max_nodes in ordered_label2num_instances.items():
+ new_value = min(num_nodes_left, int(math.ceil(num_max_nodes * reduction_factor)))
+ num_nodes_left -= new_value
+ new_scale_up_num_nodes[label] = new_value
+
+ num_total_new_scale_up_nodes = sum(new_scale_up_num_nodes.values())
+ assert num_total_new_scale_up_nodes == limit, 'Floating point imprecision detected. ' \
+ 'Num Nodes: {} Limit: {}'. \
+ format(num_total_new_scale_up_nodes, limit)
+
+ for label, num_requested_instances in label2num_instances.items():
+ num_planned_instances = new_scale_up_num_nodes.get(label, 0)
+ if num_planned_instances != num_requested_instances:
+ logging.info('Limiting upscale of %s from %d to %d', label, num_requested_instances, num_planned_instances)
+
+ return new_scale_up_num_nodes
+
+
+def _apply_downscale_limit(limit, scale_down_nodes):
+ i = 0
+ shuffeled = list(scale_down_nodes.items())
+ random.shuffle(shuffeled)
+ new_scale_down_nodes = defaultdict(list)
+ for label, nodes in shuffeled:
+ for node in nodes:
+ i += 1
+ if i > limit:
+ logging.info('Reached downscale limit')
+ break
+ else:
+ new_scale_down_nodes[label].append(node)
+
+ if i >= limit:
+ break
+
+ return new_scale_down_nodes
+
+
+def _get_jenkins_handle() -> jenkinsapi.jenkins.Jenkins: # pragma: no cover
+ from requests.exceptions import HTTPError
+
+ jenkins_credentials = _get_jenkins_credentials()
+ try:
+ server = jenkinsapi.jenkins.Jenkins(baseurl=jenkins_credentials['jenkins_url'],
+ username=jenkins_credentials['github_username'],
+ password=jenkins_credentials['github_token'],
+ timeout=JENKINS_REQUEST_TIMEOUT_SECONDS)
+ except HTTPError as e:
+ logging.exception('Error initializing Jenkins API.')
+ if e.response.status_code == 500:
+ logging.error('Did you properly set up the API token? https://t.corp.amazon.com/MXBLN-376')
+
+ logging.error('HTML response - use an HTML beautifier to view it properly: %s', e.response.content)
+ raise Exception('Error initializing Jenkins API', e)
+
+ # Jenkins returns a 302 (redirect) for certain requests. That's a valid response code somehow.....
+ # We do not want to follow redirects since they are very expensive and cause timeouts
+ # See https://t.corp.amazon.com/MXBLN-298/communication for further details
+ server.requester.VALID_STATUS_CODES.extend([302])
+ _add_timer_to_jenkins_requester(jenkins_server=server)
+ return server
+
+
+def _add_timer_to_jenkins_requester(jenkins_server):
+ """
+ Attach a hook to the Jenkins requester functions that allow to print the duration of an HTTP request
+ :param jenkins_server: Hooked jenkins server
+ :return: Nothing
+ """
+
+ def timing_decorator(f):
+ def wrap(*args, **kwargs):
+ # These guys don't like to call their API in a consistent fashion. Thus, we have to play hide and seek
+ # and search for the argument...
+ url = None
+ for arg in args:
+ if '://' in arg:
+ url = arg
+ break
+
+ if not url:
+ url = kwargs['url']
+
+ logging.debug('%s is starting request to %s', f.__name__, url)
+
+ time1 = time.time()
+ ret = f(*args, **kwargs)
+ time2 = time.time()
+
+ logging.debug('{:s} took {:.3f} ms to request {:s}'.format(f.__name__, (time2 - time1) * 1000.0, url))
+
+ return ret
+
+ return wrap
+
+ jenkins_server.requester.get_and_confirm_status = \
+ timing_decorator(jenkins_server.requester.get_and_confirm_status)
+ jenkins_server.requester.post_and_confirm_status = \
+ timing_decorator(jenkins_server.requester.post_and_confirm_status)
+ jenkins_server.requester.get_url = \
+ timing_decorator(jenkins_server.requester.get_url)
+ jenkins_server.requester.post_url = \
+ timing_decorator(jenkins_server.requester.post_url)
+
+
+@memoize
+def _get_jenkins_credentials(): # pragma: no cover
+ secret_name = os.environ['SECRET_NAME']
+ endpoint_url = os.environ['SECRET_ENDPOINT_URL']
+ region_name = os.environ['SECRET_ENDPOINT_REGION']
+
+ session = _get_aws_session()
+ client = session.client(
+ service_name='secretsmanager',
+ region_name=region_name,
+ endpoint_url=endpoint_url
+ )
+ try:
+ get_secret_value_response = client.get_secret_value(
+ SecretId=secret_name
+ )
+ except ClientError as client_error:
+ if client_error.response['Error']['Code'] == 'ResourceNotFoundException':
+ logging.exception("The requested secret %s was not found", secret_name)
+ elif client_error.response['Error']['Code'] == 'InvalidRequestException':
+ logging.exception("The request was invalid due to:")
+ elif client_error.response['Error']['Code'] == 'InvalidParameterException':
+ logging.exception("The request had invalid params:")
+ else:
+ raise
+ else:
+ secret = get_secret_value_response['SecretString']
+ secret_dict = json.loads(secret)
+ return secret_dict
+
+
+@memoize
+def _warm_pool_node_counts() -> Dict[str, int]:
+ """
+ Leave some buffer for warm pool. This code makes sure to always leave X instances in idle while scaling down.
+ For example: 5 instances running, 3 in idle, WARM_POOL_SIZE set to 2. This code will remove only 1 instance,
+ leading to 4 instances running and 2 in idle.
+ :return: Config dict
+ """
+ return json.loads(os.environ['WARM_POOL_SIZE'])
+
+
+@memoize
+def _minimum_queue_times_sec() -> Dict[str, int]:
+ """
+ Retrieves values specifying the minimum amount of time a job should remain in the Jenkins queue before being
+ considered in scale_up calls. Each job type is configured dependent on the required jenkins node label.
+ :return: A dictionary with values representing minimum queue time required before a job is considered in scaling,
+ and a key representing the jenkins node label
+ """
+ return json.loads(os.environ['MINIMUM_QUEUE_TIMES_SEC'])
+
+
+@memoize
+def _maximum_startup_time() -> Dict[str, int]:
+ """
+ Retrieve the maximum time an instance is allowed to need for start up.
+ :return: Dictionary mapping maximum startup time in seconds to the appropriate label
+ """
+ return json.loads(os.environ['MAXIMUM_STARTUP_TIME_SEC'])
+
+
+@memoize
+def _ignored_jenkins_node_names() -> List[str]:
+ """
+ Ignore nodes with these names
+ :return: Config list
+ """
+ return json.loads(os.environ['IGNORED_JENKINS_NODE_NAMES'])
+
+
+@memoize
+def _ignored_jenkins_node_labels() -> List[str]:
+ """
+ Labels that are ignored by the auto scaling system. The system will not throw an error if they are encountered
+ :return: Config list
+ """
+ return json.loads(os.environ['IGNORED_JENKINS_NODE_LABELS'])
+
+
+@memoize
+def _managed_jenkins_node_labels() -> List[str]:
+ """
+ Get a list of labels that are being managed by the auto scaling system
+ :return: Config list
+ """
+ return json.loads(os.environ['MANAGED_JENKINS_NODE_LABELS'])
+
+
+@memoize
+def _ccache_efs_addresses() -> Dict[str, str]:
+ """
+ Get a dict mapping the ccache EFS DNS addresses to their labels
+ :return: Config dict
+ """
+ return json.loads(os.environ['CCACHE_EFS_DNS'])
+
+
+@memoize
+def _launch_templates():
+ return json.loads(os.environ['LAUNCH_TEMPLATES'])
+
+
+@memoize
+def _get_nb_executors_per_label():
+ return json.loads(os.environ['EXECUTORS_PER_LABEL'])
+
+
+@memoize
+def _get_jenkins_private_tunnel_address():
+ return os.environ['JENKINS_PRIV_TUNNEL']
+
+
+@memoize
+def _get_slave_configuration():
+ return {
+ 'mxnetlinux-cpu': {
+ 'num_executors': _get_nb_executors_per_label()['mxnetlinux-cpu'], # Number of executors
+ 'node_description': '[AUTOSCALING] MXNet slave running Ubuntu 16.04 on a c5.18xlarge',
+ 'remote_fs': '/home/jenkins_slave', # Remote workspace location
+ 'labels': 'mxnetlinux-cpu', # Space separated labels string
+ 'exclusive': True, # Only run jobs assigned to it
+ 'tunnel': _get_jenkins_private_tunnel_address(),
+ 'job_name_restriction_regex': '^(?!restricted-).+' # Run only unrestricted jobs
+ },
+ 'mxnetlinux-gpu': {
+ 'num_executors': _get_nb_executors_per_label()['mxnetlinux-gpu'], # Number of executors
+ 'node_description': '[AUTOSCALING] MXNet slave running Ubuntu 16.04 on a g3.8xlarge',
+ 'remote_fs': '/home/jenkins_slave', # Remote workspace location
+ 'labels': 'mxnetlinux-gpu', # Space separated labels string
+ 'exclusive': True, # Only run jobs assigned to it
+ 'tunnel': _get_jenkins_private_tunnel_address(),
+ 'job_name_restriction_regex': '^(?!restricted-).+' # Run only unrestricted jobs
+ },
+ 'restricted-mxnetlinux-gpu': {
+ 'num_executors': _get_nb_executors_per_label()['restricted-mxnetlinux-gpu'], # Number of executors
+ 'node_description': '[AUTOSCALING] MXNet slave running Ubuntu 16.04 on a g3.8xlarge',
+ 'remote_fs': '/home/jenkins_slave', # Remote workspace location
+ 'labels': 'restricted-mxnetlinux-gpu', # Space separated labels string
+ 'exclusive': True, # Only run jobs assigned to it
+ 'tunnel': _get_jenkins_private_tunnel_address(),
+ 'job_name_restriction_regex': '^restricted-(.*)' # Only run jobs which start with restricted-
+ },
+ 'mxnetlinux-gpu-p3': {
+ 'num_executors': _get_nb_executors_per_label()['mxnetlinux-gpu-p3'], # Number of executors
+ 'node_description': '[AUTOSCALING] MXNet slave running Ubuntu 16.04 on a p3.2xlarge',
+ 'remote_fs': '/home/jenkins_slave', # Remote workspace location
+ 'labels': 'mxnetlinux-gpu-p3', # Space separated labels string
+ 'exclusive': True, # Only run jobs assigned to it
+ 'tunnel': _get_jenkins_private_tunnel_address(),
+ 'job_name_restriction_regex': '^(?!restricted-).+' # Run only unrestricted jobs
+ },
+ 'restricted-mxnetlinux-gpu-p3': {
+ 'num_executors': _get_nb_executors_per_label()['mxnetlinux-gpu-p3'], # Number of executors
+ 'node_description': '[AUTOSCALING] MXNet slave running Ubuntu 16.04 on a p3.2xlarge (restricted)',
+ 'remote_fs': '/home/jenkins_slave', # Remote workspace location
+ 'labels': 'restricted-mxnetlinux-gpu-p3', # Space separated labels string
+ 'exclusive': True, # Only run jobs assigned to it
+ 'tunnel': _get_jenkins_private_tunnel_address(),
+ 'job_name_restriction_regex': '^restricted-(.*)' # Only run jobs which start with restricted-
+ },
+ 'mxnetlinux-gpu-p3-8xlarge': {
+ 'num_executors': _get_nb_executors_per_label()['mxnetlinux-gpu-p3-8xlarge'], # Number of executors
+ 'node_description': '[AUTOSCALING] MXNet slave running Ubuntu 16.04 on a p3.8xlarge',
+ 'remote_fs': '/home/jenkins_slave', # Remote workspace location
+ 'labels': 'mxnetlinux-gpu-p3-8xlarge', # Space separated labels string
+ 'exclusive': True, # Only run jobs assigned to it
+ 'tunnel': _get_jenkins_private_tunnel_address(),
+ 'job_name_restriction_regex': '^(?!restricted-).+' # Run only unrestricted jobs
+ },
+ 'mxnetwindows-cpu': {
+ 'num_executors': _get_nb_executors_per_label()['mxnetwindows-cpu'], # Number of executors
+ 'node_description': '[AUTOSCALING] MXNet slave running Windows Datacenter 2016 on a c5.18xlarge',
+ 'remote_fs': 'C:/jenkins_slave', # Remote workspace location
+ 'labels': 'mxnetwindows-cpu', # Space separated labels string
+ 'exclusive': True, # Only run jobs assigned to it
+ 'tunnel': _get_jenkins_private_tunnel_address(),
+ 'job_name_restriction_regex': '^(?!restricted-).+' # Run only unrestricted jobs
+ },
+ 'mxnetwindows-gpu': {
+ 'num_executors': _get_nb_executors_per_label()['mxnetwindows-gpu'], # Number of executors
+ 'node_description': '[AUTOSCALING] MXNet slave running Windows Datacenter 2016 on a g3.8xlarge',
+ 'remote_fs': 'C:/jenkins_slave', # Remote workspace location
+ 'labels': 'mxnetwindows-gpu', # Space separated labels string
+ 'exclusive': True, # Only run jobs assigned to it
+ 'tunnel': _get_jenkins_private_tunnel_address(),
+ 'job_name_restriction_regex': '^(?!restricted-).+' # Run only unrestricted jobs
+ },
+ 'restricted-mxnetlinux-cpu': {
+ 'num_executors': _get_nb_executors_per_label()['mxnetlinux-cpu'], # Number of executors
+ 'node_description': '[AUTOSCALING] MXNet slave running Ubuntu 16.04 on a c5.18xlarge',
+ 'remote_fs': '/home/jenkins_slave', # Remote workspace location
+ 'labels': 'restricted-mxnetlinux-cpu', # Space separated labels string
+ 'exclusive': True, # Only run jobs assigned to it
+ 'tunnel': _get_jenkins_private_tunnel_address(),
+ 'job_name_restriction_regex': '^restricted-(.*)' # Only run jobs which start with restricted-
+ },
+ 'utility': {
+ 'num_executors': _get_nb_executors_per_label()['utility'], # Number of executors
+ 'node_description': '[AUTOSCALING] Slave for utility operations',
+ 'remote_fs': '/home/jenkins_slave', # Remote workspace location
+ 'labels': 'utility', # Space separated labels string
+ 'exclusive': True, # Only run jobs assigned to it
+ 'tunnel': _get_jenkins_private_tunnel_address(),
+ 'job_name_restriction_regex': '^(?!restricted-).+' # Run only unrestricted jobs
+ },
+ 'restricted-utility': {
+ 'num_executors': _get_nb_executors_per_label()['restricted-utility'], # Number of executors
+ 'node_description': '[AUTOSCALING] Restricted slave for utility operations',
+ 'remote_fs': '/home/jenkins_slave', # Remote workspace location
+ 'labels': 'restricted-utility', # Space separated labels string
+ 'exclusive': True, # Only run jobs assigned to it
+ 'tunnel': _get_jenkins_private_tunnel_address(),
+ 'job_name_restriction_regex': '^restricted-(.*)' # Only run jobs which start with restricted-
+ }
+ }
+
+
+@memoize
+def _get_aws_session() -> boto3.Session: # pragma: no cover
+ """
+ Get the boto3 AWS session
+ :return: Session object
+ """
+ # For local debugging:
+ # session = boto3.Session(profile_name=os.environ['AWS_PROFILE'], region_name=os.environ['AWS_REGION'])
+ session = boto3.Session()
+ return session
+
+
+def _get_log_level(env_var_key, default_level) -> int:
+ """
+ Read the log level from an environment variable or set it to a default if it does not exist
+ :param env_var_key:
+ :param default_level:
+ :return:
+ """
+ try:
+ logging_level = os.environ[env_var_key]
+ except KeyError:
+ logging.warning('Unable to find %s env var. Defaulting to %s.', env_var_key, default_level)
+ return default_level
+ else:
+ if logging_level == 'DEBUG':
+ return logging.DEBUG
+ elif logging_level == 'INFO':
+ return logging.INFO
+ elif logging_level == 'WARNING':
+ return logging.WARNING
+ elif logging_level == 'ERROR':
+ return logging.ERROR
+ else:
+ raise KeyError('Unable to match logging level {} for {}'.format(logging_level, env_var_key))
+
+
+def _ec2Instance_tag_dict(ec2_object):
+ """Given an tagable ec2_object, return dictionary of existing tags."""
+ tag_dict = {}
+ if ec2_object.tags is None:
+ return tag_dict
+
+ for tag in ec2_object.tags:
+ tag_dict[tag['Key']] = tag['Value']
+ return tag_dict
+
+
+def chunks(source_list, chunk_size):
+ """return list with n chunks from l."""
+ chunk_size = max(1, chunk_size)
+ return (source_list[i:i + chunk_size] for i in range(0, len(source_list), chunk_size))
+
+
+def _merge_dicts_nested_lists(dict1: Dict[Any, List[Any]], dict2: Dict[Any, List[Any]]) -> Dict[Any, List[Any]]:
+ """
+ Merge two dicts that contain lists as values.
+ :param dict1: Dict1
+ :param dict2: Dict2
+ :return: Merged dict
+ """
+ result_dict: Dict[Any, List[Any]] = dict()
+ # Determine key matches which require list merges
+ if dict1 and dict2:
+ matching_keys = set(dict1.keys()).intersection(set(dict2.keys()))
+ else:
+ matching_keys = set()
+
+ for key in matching_keys:
+ result_dict[key] = list(dict1[key] + dict2[key])
+
+ # Remainders from dict1
+ if dict1:
+ for key in set(dict1.keys() - matching_keys):
+ result_dict[key] = list(dict1[key])
+
+ # Remainders from dict2
+ if dict2:
+ for key in set(dict2.keys() - matching_keys):
+ result_dict[key] = list(dict2[key])
+
+ return result_dict
+
+
+def scaling(): # pragma: no cover
+ """
+ Main handler, used by the lambda function
+ :return: None
+ """
+ # All underlying methods are being unit tested. This function will have to be integration tested in a live dev
+ # environment.
+ logging.getLogger().setLevel(_get_log_level('LOGGING_LEVEL', logging.INFO))
+ logging.getLogger('botocore').setLevel(_get_log_level('LOGGING_LEVEL_BOTOCORE', logging.INFO))
+ logging.getLogger('boto3').setLevel(_get_log_level('LOGGING_LEVEL_BOTO3', logging.INFO))
+ logging.getLogger('urllib3').setLevel(_get_log_level('LOGGING_LEVEL_URLLIB3', logging.INFO))
+ logging.getLogger('requests').setLevel(_get_log_level('LOGGING_LEVEL_REQUESTS', logging.ERROR))
+ logging.getLogger('botocore.vendored.requests.packages.urllib3.connectionpool').setLevel(logging.ERROR)
+ logging.getLogger('jenkinsapi.node').setLevel(logging.INFO)
+
+ boto_config = Config(
+ retries=dict(
+ max_attempts=0 # Don't retry but fail fast
+ )
+ )
+ jenkins = _get_jenkins_handle()
+ aws_session = _get_aws_session()
+ ec2_resource = aws_session.resource('ec2', config=boto_config)
+
+ # list of jenkinsapi.nodes.Node
+ nodes = jenkins.get_nodes()._data['computer']
+ logging.info("Found %d nodes registered in Jenkins.", len(nodes))
+
+ # Ec2 instances
+ instance_uptime = _instance_uptime(ec2_resource=ec2_resource)
+ logging.info("Found %d ec2 instances.", len(instance_uptime))
+
+ if len(instance_uptime) != len(nodes):
+ logging.warning("nodes and instances don't have the same length.")
+
+ unconnected_label2instance_names = _unconnected_instances(
+ nodes=nodes,
+ instance_uptime=instance_uptime,
+ ec2_resource=ec2_resource)
+
+ queue_items = jenkins.get_queue()._data['items']
+
+ label2num_instances = determine_scale_up_nodes(
+ queue_items=queue_items, nodes=nodes, unconnected=unconnected_label2instance_names)
+
+ scale_down_nodes = determine_scale_down_nodes(nodes_data=nodes,
+ instance_uptime=instance_uptime)
+
+ ############################################
+ # Detection of instances and slots to be cleaned up
+ (label2faulty_nodes, orphaned_instances) = _determine_faulty_nodes(nodes=nodes,
+ instance_uptime=instance_uptime,
+ unconnected_instances=unconnected_label2instance_names)
+ if label2faulty_nodes:
+ faulty = []
+ for faulty_nodes in label2faulty_nodes.values():
+ faulty.extend([node['displayName'] for node in faulty_nodes])
+ logging.warning('Found %d faulty instances: %s', len(faulty), faulty)
+
+ if orphaned_instances:
+ logging.error('Found %d orphaned instances: %s', len(orphaned_instances), orphaned_instances)
+ ############################################
+
+ label2num_instances = _apply_upscale_limit(limit=NUM_UPSCALES_PER_ROUND, label2num_instances=label2num_instances)
+ scale_down_nodes = _apply_downscale_limit(limit=NUM_DOWNSCALES_PER_ROUND, scale_down_nodes=scale_down_nodes)
+ scale_down_nodes = _merge_dicts_nested_lists(scale_down_nodes, label2faulty_nodes)
+
+ execute_scale_down_logic(
+ jenkins_server=jenkins,
+ ec2_resource=ec2_resource,
+ scale_down_nodes=scale_down_nodes,
+ )
+ ############
+ _terminate_ec2_instances(orphaned_instances, ec2_resource)
+ ############
+
+ execute_scale_up_logic(
+ jenkins_server=jenkins,
+ ec2_resource=ec2_resource,
+ scale_up_nb_nodes=label2num_instances)
+
+
+# pylint: disable=unused-argument
+def scaling_handler(event, context): # pragma: no cover
+ """
+ Handler to be called by lambda
+ :param event:
+ :param context:
+ :return:
+ """
+ try:
+ scaling()
+ except Exception: # pylint: disable=broad-except
+ logging.exception('Unexpected exception')
+ logging.fatal('Unexpected exception')
+ # This try-catch is important because we have to catch all exceptions. Otherwise, the exceptions bubble up to
+ # lambda and the service retries executing multiple times. We only want exactly one execution per request.
+
+
+if __name__ == "__main__":
+ logging.basicConfig(level=logging.DEBUG)
+ scaling_handler(None, None)
diff --git a/services/jenkins-autoscaling/lambda_mxnet_ci/autoscaling/requirements.txt b/services/jenkins-autoscaling/lambda_mxnet_ci/autoscaling/requirements.txt
new file mode 100644
index 0000000..233d5ae
--- /dev/null
+++ b/services/jenkins-autoscaling/lambda_mxnet_ci/autoscaling/requirements.txt
@@ -0,0 +1,4 @@
+boto3==1.7.63
+botocore==1.10.63
+jenkinsapi==0.3.6
+joblib==0.12.1
diff --git a/services/jenkins-autoscaling/lambda_mxnet_ci/autoscaling/serverless.yml b/services/jenkins-autoscaling/lambda_mxnet_ci/autoscaling/serverless.yml
new file mode 100644
index 0000000..c3e292b
--- /dev/null
+++ b/services/jenkins-autoscaling/lambda_mxnet_ci/autoscaling/serverless.yml
@@ -0,0 +1,53 @@
+service: mxnet-ci-master-scaling
+
+frameworkVersion: ">=1.2.0 <2.0.0"
+
+provider:
+ name: aws
+ runtime: python3.6
+ region: us-west-2
+ memorySize: 128
+ timeout: 290
+ stage: ${opt:stage}
+ environment: ${file(environment.yml):${self:provider.stage}}
+ iamRoleStatements:
+ - Effect: "Allow"
+ Action:
+ - "secretsmanager:DescribeSecret"
+ - "secretsmanager:GetSecretValue"
+ Resource:
+ - ${self:provider.environment.SECRETS_MANAGER_ARN}
+ - Effect: "Allow"
+ Action:
+ - "ec2:Describe*"
+ Resource:
+ - "*"
+ - Effect: "Allow"
+ Action:
+ - "ec2:RunInstances"
+ - "ec2:TerminateInstances"
+ - "ec2:CreateTags"
+ Resource:
+ - ${self:provider.environment.EC2_ACCOUNT_ARN}
+ - "arn:aws:ec2:us-west-2::image/ami-*"
+ - Effect: "Allow"
+ Action:
+ - "iam:PassRole"
+ Resource:
+ - ${self:provider.environment.IAM_JENKINS_SLAVE_ROLE}
+ - ${self:provider.environment.IAM_JENKINS_RESTRICTED_SLAVE_ROLE}
+
+functions:
+ mxnet-ci-master-scaling:
+ handler: handler.scaling_handler
+ reservedConcurrency: 1
+ events:
+ # Invoke Lambda function on a schedule
+ - schedule: rate(5 minutes)
+
+custom:
+ pythonRequirements:
+ dockerizePip: true
+
+plugins:
+ - serverless-python-requirements
diff --git a/services/jenkins-autoscaling/lambda_mxnet_ci/pylintrc b/services/jenkins-autoscaling/lambda_mxnet_ci/pylintrc
new file mode 100644
index 0000000..9cbc58b
--- /dev/null
+++ b/services/jenkins-autoscaling/lambda_mxnet_ci/pylintrc
@@ -0,0 +1,415 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[MASTER]
+
+# Specify a configuration file.
+#rcfile=
+
+# Python code to execute, usually for sys.path manipulation such as
+# pygtk.require().
+#init-hook=
+
+# Add files or directories to the blacklist. They should be base names, not
+# paths.
+ignore=CVS
+
+# Add files or directories matching the regex patterns to the blacklist. The
+# regex matches against base names, not paths.
+ignore-patterns=
+
+# Pickle collected data for later comparisons.
+persistent=yes
+
+# List of plugins (as comma separated values of python modules names) to load,
+# usually to register additional checkers.
+load-plugins=
+
+# Use multiple processes to speed up Pylint.
+jobs=8
+
+# Allow loading of arbitrary C extensions. Extensions are imported into the
+# active Python interpreter and may run arbitrary code.
+unsafe-load-any-extension=no
+
+# A comma-separated list of package or module names from where C extensions may
+# be loaded. Extensions are loading into the active Python interpreter and may
+# run arbitrary code
+extension-pkg-whitelist=
+
+# Allow optimization of some AST trees. This will activate a peephole AST
+# optimizer, which will apply various small optimizations. For instance, it can
+# be used to obtain the result of joining multiple strings with the addition
+# operator. Joining a lot of strings can lead to a maximum recursion error in
+# Pylint and this flag can prevent that. It has one side effect, the resulting
+# AST will be different than the one from reality. This option is deprecated
+# and it will be removed in Pylint 2.0.
+optimize-ast=no
+
+
+[MESSAGES CONTROL]
+
+# Only show warnings with the listed confidence levels. Leave empty to show
+# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED
+confidence=
+
+# Enable the message, report, category or checker with the given id(s). You can
+# either give multiple identifier separated by comma (,) or put this option
+# multiple time (only on the command line, not in the configuration file where
+# it should appear only once). See also the "--disable" option for examples.
+#enable=
+
+# Disable the message, report, category or checker with the given id(s). You
+# can either give multiple identifiers separated by comma (,) or put this
+# option multiple times (only on the command line, not in the configuration
+# file where it should appear only once).You can also use "--disable=all" to
+# disable everything first and then reenable specific checks. For example, if
+# you want to run only the similarities checker, you can use "--disable=all
+# --enable=similarities". If you want to run only the classes checker, but have
+# no Warning level messages displayed, use"--disable=all --enable=classes
+# --disable=W"
+disable=protected-access,print-statement,parameter-unpacking,unpacking-in-except,old-raise-syntax,backtick,import-star-module-level,apply-builtin,basestring-builtin,buffer-builtin,cmp-builtin,coerce-builtin,execfile-builtin,file-builtin,long-builtin,raw_input-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,no-absolute-import,old-division,dict-iter-method,dict-view-method,next-method-called,metaclass [...]
+
+
+[REPORTS]
+
+# Set the output format. Available formats are text, parseable, colorized, msvs
+# (visual studio) and html. You can also give a reporter class, eg
+# mypackage.mymodule.MyReporterClass.
+output-format=text
+
+# Put messages in a separate file for each module / package specified on the
+# command line instead of printing them on stdout. Reports (if any) will be
+# written in a file name "pylint_global.[txt|html]". This option is deprecated
+# and it will be removed in Pylint 2.0.
+files-output=no
+
+# Tells whether to display a full report or only the messages
+reports=yes
+
+# Python expression which should return a note less than 10 (10 is the highest
+# note). You have access to the variables errors warning, statement which
+# respectively contain the number of errors / warnings messages and the total
+# number of statements analyzed. This is used by the global evaluation report
+# (RP0004).
+evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)
+
+# Template used to display messages. This is a python new-style format string
+# used to format the message information. See doc for all details
+#msg-template=
+
+
+[BASIC]
+
+# Good variable names which should always be accepted, separated by a comma
+good-names=i,j,k,ex,Run,_
+
+# Bad variable names which should always be refused, separated by a comma
+bad-names=foo,bar,baz,toto,tutu,tata
+
+# Colon-delimited sets of names that determine each other's naming style when
+# the name regexes allow several styles.
+name-group=
+
+# Include a hint for the correct naming format with invalid-name
+include-naming-hint=no
+
+# List of decorators that produce properties, such as abc.abstractproperty. Add
+# to this list to register other decorators that produce valid properties.
+property-classes=abc.abstractproperty
+
+# Regular expression matching correct module names
+module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$
+
+# Naming hint for module names
+module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$
+
+# Regular expression matching correct constant names
+const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$
+
+# Naming hint for constant names
+const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$
+
+# Regular expression matching correct class names
+class-rgx=[A-Z_][a-zA-Z0-9]+$
+
+# Naming hint for class names
+class-name-hint=[A-Z_][a-zA-Z0-9]+$
+
+# Regular expression matching correct function names
+function-rgx=[a-z_][a-z0-9_]{2,40}$
+
+# Naming hint for function names
+function-name-hint=[a-z_][a-z0-9_]{2,40}$
+
+# Regular expression matching correct method names
+method-rgx=[a-z_][a-z0-9_]{2,40}$
+
+# Naming hint for method names
+method-name-hint=[a-z_][a-z0-9_]{2,40}$
+
+# Regular expression matching correct attribute names
+attr-rgx=[a-z_][a-z0-9_]{2,40}$
+
+# Naming hint for attribute names
+attr-name-hint=[a-z_][a-z0-9_]{2,40}$
+
+# Regular expression matching correct argument names
+argument-rgx=[a-z_][a-z0-9_]{2,40}$
+
+# Naming hint for argument names
+argument-name-hint=[a-z_][a-z0-9_]{2,40}$
+
+# Regular expression matching correct variable names
+variable-rgx=[a-z_][a-z0-9_]{2,40}$
+
+# Naming hint for variable names
+variable-name-hint=[a-z_][a-z0-9_]{2,40}$
+
+# Regular expression matching correct class attribute names
+class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,40}|(__.*__))$
+
+# Naming hint for class attribute names
+class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,40}|(__.*__))$
+
+# Regular expression matching correct inline iteration names
+inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$
+
+# Naming hint for inline iteration names
+inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$
+
+# Regular expression which should only match function or class names that do
+# not require a docstring.
+#no-docstring-rgx=^_
+no-docstring-rgx=.
+
+# Minimum line length for functions/classes that require docstrings, shorter
+# ones are exempt.
+docstring-min-length=-1
+
+
+[ELIF]
+
+# Maximum number of nested blocks for function / method body
+max-nested-blocks=5
+
+
+[FORMAT]
+
+# Maximum number of characters on a single line.
+max-line-length=140
+
+# Regexp for a line that is allowed to be longer than the limit.
+ignore-long-lines=^\s*(# )?<?https?://\S+>?$
+
+# Allow the body of an if to be on the same line as the test if there is no
+# else.
+single-line-if-stmt=no
+
+# List of optional constructs for which whitespace checking is disabled. `dict-
+# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}.
+# `trailing-comma` allows a space between comma and closing bracket: (a, ).
+# `empty-line` allows space-only lines.
+no-space-check=trailing-comma,dict-separator
+
+# Maximum number of lines in a module
+max-module-lines=10000
+
+# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1
+# tab).
+indent-string=' '
+
+# Number of spaces of indent required inside a hanging or continued line.
+indent-after-paren=4
+
+# Expected format of line ending, e.g. empty (any line ending), LF or CRLF.
+expected-line-ending-format=
+
+
+[LOGGING]
+
+# Logging modules to check that the string format arguments are in logging
+# function parameter format
+logging-modules=logging
+
+
+[MISCELLANEOUS]
+
+# List of note tags to take in consideration, separated by a comma.
+notes=FIXME,XXX,TODO
+
+
+[SIMILARITIES]
+
+# Minimum lines number of a similarity.
+min-similarity-lines=4
+
+# Ignore comments when computing similarities.
+ignore-comments=yes
+
+# Ignore docstrings when computing similarities.
+ignore-docstrings=yes
+
+# Ignore imports when computing similarities.
+ignore-imports=no
+
+
+[SPELLING]
+
+# Spelling dictionary name. Available dictionaries: none. To make it working
+# install python-enchant package.
+spelling-dict=
+
+# List of comma separated words that should not be checked.
+spelling-ignore-words=
+
+# A path to a file that contains private dictionary; one word per line.
+spelling-private-dict-file=
+
+# Tells whether to store unknown words to indicated private dictionary in
+# --spelling-private-dict-file option instead of raising a message.
+spelling-store-unknown-words=no
+
+
+[TYPECHECK]
+
+# Tells whether missing members accessed in mixin class should be ignored. A
+# mixin class is detected if its name ends with "mixin" (case insensitive).
+ignore-mixin-members=yes
+
+# List of class names for which member attributes should not be checked (useful
+# for classes with dynamically set attributes). This supports the use of
+# qualified names.
+ignored-classes=optparse.Values,thread._local,_thread._local
+
+# List of members which are set dynamically and missed by pylint inference
+# system, and so shouldn't trigger E1101 when accessed. Python regular
+# expressions are accepted.
+generated-members=
+
+# List of decorators that produce context managers, such as
+# contextlib.contextmanager. Add to this list to register other decorators that
+# produce valid context managers.
+contextmanager-decorators=contextlib.contextmanager
+
+
+[VARIABLES]
+
+# Tells whether we should check for unused import in __init__ files.
+init-import=no
+
+# A regular expression matching the name of dummy variables (i.e. expectedly
+# not used).
+dummy-variables-rgx=(_+[a-zA-Z0-9]*?$)|dummy
+
+# List of additional names supposed to be defined in builtins. Remember that
+# you should avoid to define new builtins when possible.
+additional-builtins=
+
+# List of strings which can identify a callback function by name. A callback
+# name must start or end with one of those strings.
+callbacks=cb_,_cb
+
+# List of qualified module names which can have objects that can redefine
+# builtins.
+redefining-builtins-modules=six.moves,future.builtins
+
+
+[CLASSES]
+
+# List of method names used to declare (i.e. assign) instance attributes.
+defining-attr-methods=__init__,__new__,setUp
+
+# List of valid names for the first argument in a class method.
+valid-classmethod-first-arg=cls
+
+# List of valid names for the first argument in a metaclass class method.
+valid-metaclass-classmethod-first-arg=mcs
+
+
+[DESIGN]
+
+# Maximum number of arguments for function / method
+max-args=10
+
+# Argument names that match this expression will be ignored. Default to name
+# with leading underscore
+ignored-argument-names=_.*
+
+# Maximum number of locals for function / method body
+max-locals=15
+
+# Maximum number of return / yield for function / method body
+max-returns=6
+
+# Maximum number of branch for function / method body
+max-branches=12
+
+# Maximum number of statements in function / method body
+max-statements=50
+
+# Maximum number of parents for a class (see R0901).
+max-parents=7
+
+# Maximum number of attributes for a class (see R0902).
+max-attributes=7
+
+# Minimum number of public methods for a class (see R0903).
+min-public-methods=2
+
+# Maximum number of public methods for a class (see R0904).
+max-public-methods=20
+
+# Maximum number of boolean expressions in a if statement
+max-bool-expr=5
+
+
+[IMPORTS]
+
+# Deprecated modules which should not be used, separated by a comma
+deprecated-modules=optparse
+
+# Create a graph of every (i.e. internal and external) dependencies in the
+# given file (report RP0402 must not be disabled)
+import-graph=
+
+# Create a graph of external dependencies in the given file (report RP0402 must
+# not be disabled)
+ext-import-graph=
+
+# Create a graph of internal dependencies in the given file (report RP0402 must
+# not be disabled)
+int-import-graph=
+
+# Force import order to recognize a module as part of the standard
+# compatibility libraries.
+known-standard-library=
+
+# Force import order to recognize a module as part of a third party library.
+known-third-party=enchant
+
+# Analyse import fallback blocks. This can be used to support both Python 2 and
+# 3 compatible code, which means that the block might have code that exists
+# only in one or another interpreter, leading to false positives when analysed.
+analyse-fallback-blocks=no
+
+
+[EXCEPTIONS]
+
+# Exceptions that will emit a warning when being caught. Defaults to
+# "Exception"
+overgeneral-exceptions=Exception
diff --git a/services/jenkins-autoscaling/lambda_mxnet_ci/tests/.coveragerc b/services/jenkins-autoscaling/lambda_mxnet_ci/tests/.coveragerc
new file mode 100644
index 0000000..d7ad1fc
--- /dev/null
+++ b/services/jenkins-autoscaling/lambda_mxnet_ci/tests/.coveragerc
@@ -0,0 +1,44 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# .coveragerc to control coverage.py
+[run]
+branch = True
+
+
+[report]
+# Regexes for lines to exclude from consideration
+exclude_lines =
+ # Have to re-enable the standard pragma
+ pragma: no cover
+
+ # Don't complain about missing debug-only code:
+ def __repr__
+ if self\.debug
+
+ # Don't complain if tests don't hit defensive assertion code:
+ raise AssertionError
+ raise NotImplementedError
+
+ # Don't complain if non-runnable code isn't run:
+ if 0:
+ if __name__ == .__main__.:
+
+ignore_errors = True
+
+[html]
+directory = coverage_html_report
diff --git a/services/jenkins-autoscaling/lambda_mxnet_ci/tests/.gitignore b/services/jenkins-autoscaling/lambda_mxnet_ci/tests/.gitignore
new file mode 100644
index 0000000..6b46151
--- /dev/null
+++ b/services/jenkins-autoscaling/lambda_mxnet_ci/tests/.gitignore
@@ -0,0 +1,3 @@
+# Code coverage files
+.coverage
+cover
diff --git a/services/jenkins-autoscaling/lambda_mxnet_ci/tests/placebo/test_autoscaling/test_get_starting_nodes/ec2.DescribeInstances_1.json b/services/jenkins-autoscaling/lambda_mxnet_ci/tests/placebo/test_autoscaling/test_get_starting_nodes/ec2.DescribeInstances_1.json
new file mode 100644
index 0000000..72c55c1
--- /dev/null
+++ b/services/jenkins-autoscaling/lambda_mxnet_ci/tests/placebo/test_autoscaling/test_get_starting_nodes/ec2.DescribeInstances_1.json
@@ -0,0 +1,361 @@
+{
+ "status_code": 200,
+ "data": {
+ "Reservations": [
+ {
+ "Groups": [],
+ "Instances": [
+ {
+ "AmiLaunchIndex": 0,
+ "ImageId": "ami-20e17d58",
+ "InstanceId": "i-000000000000001",
+ "InstanceType": "g3.8xlarge",
+ "KeyName": "REDACTED",
+ "LaunchTime": {
+ "__class__": "datetime",
+ "year": 2018,
+ "month": 3,
+ "day": 20,
+ "hour": 15,
+ "minute": 35,
+ "second": 16,
+ "microsecond": 0
+ },
+ "Monitoring": {
+ "State": "disabled"
+ },
+ "Placement": {
+ "AvailabilityZone": "us-west-2b",
+ "GroupName": "",
+ "Tenancy": "default"
+ },
+ "PrivateDnsName": "ip-172-31-30-54.us-west-2.compute.internal",
+ "PrivateIpAddress": "172.31.30.54",
+ "ProductCodes": [],
+ "PublicDnsName": "ec2-34-209-91-146.us-west-2.compute.amazonaws.com",
+ "PublicIpAddress": "34.209.91.146",
+ "State": {
+ "Code": 16,
+ "Name": "running"
+ },
+ "StateTransitionReason": "",
+ "SubnetId": "subnet-05d5d863",
+ "VpcId": "vpc-02c00d7b",
+ "Architecture": "x86_64",
+ "BlockDeviceMappings": [],
+ "ClientToken": "",
+ "EbsOptimized": true,
+ "EnaSupport": true,
+ "Hypervisor": "xen",
+ "IamInstanceProfile": {
+ "Arn": "arn:aws:iam::REDACTED:instance-profile/jenkins_slave_profile",
+ "Id": "AIPAIJA6M2HUUH6EQTA44"
+ },
+ "NetworkInterfaces": [],
+ "RootDeviceName": "/dev/sda1",
+ "RootDeviceType": "ebs",
+ "SecurityGroups": [],
+ "SourceDestCheck": true,
+ "Tags": [
+ {
+ "Key": "Name",
+ "Value": "mxnetlinux-gpu10"
+ },
+ {
+ "Key": "label",
+ "Value": "mxnetlinux-gpu"
+ },
+ {
+ "Key": "AutoScaledSlave",
+ "Value": "True"
+ }
+ ],
+ "VirtualizationType": "hvm"
+ },
+ {
+ "AmiLaunchIndex": 0,
+ "ImageId": "ami-20e17d58",
+ "InstanceId": "i-000000000000002",
+ "InstanceType": "g3.8xlarge",
+ "KeyName": "REDACTED",
+ "LaunchTime": {
+ "__class__": "datetime",
+ "year": 2018,
+ "month": 3,
+ "day": 20,
+ "hour": 15,
+ "minute": 35,
+ "second": 16,
+ "microsecond": 0
+ },
+ "Monitoring": {
+ "State": "disabled"
+ },
+ "Placement": {
+ "AvailabilityZone": "us-west-2b",
+ "GroupName": "",
+ "Tenancy": "default"
+ },
+ "PrivateDnsName": "ip-172-31-30-54.us-west-2.compute.internal",
+ "PrivateIpAddress": "172.31.30.54",
+ "ProductCodes": [],
+ "PublicDnsName": "ec2-34-209-91-146.us-west-2.compute.amazonaws.com",
+ "PublicIpAddress": "34.209.91.146",
+ "State": {
+ "Code": 16,
+ "Name": "running"
+ },
+ "StateTransitionReason": "",
+ "SubnetId": "subnet-05d5d863",
+ "VpcId": "vpc-02c00d7b",
+ "Architecture": "x86_64",
+ "BlockDeviceMappings": [],
+ "ClientToken": "",
+ "EbsOptimized": true,
+ "EnaSupport": true,
+ "Hypervisor": "xen",
+ "IamInstanceProfile": {
+ "Arn": "arn:aws:iam::REDACTED:instance-profile/jenkins_slave_profile",
+ "Id": "AIPAIJA6M2HUUH6EQTA44"
+ },
+ "NetworkInterfaces": [],
+ "RootDeviceName": "/dev/sda1",
+ "RootDeviceType": "ebs",
+ "SecurityGroups": [],
+ "SourceDestCheck": true,
+ "Tags": [
+ {
+ "Key": "Name",
+ "Value": "mxnetlinux-gpu11"
+ },
+ {
+ "Key": "label",
+ "Value": "mxnetlinux-gpu"
+ },
+ {
+ "Key": "AutoScaledSlave",
+ "Value": "True"
+ }
+ ],
+ "VirtualizationType": "hvm"
+ },
+ {
+ "AmiLaunchIndex": 0,
+ "ImageId": "ami-20e17d58",
+ "InstanceId": "i-000000000000003",
+ "InstanceType": "g3.8xlarge",
+ "KeyName": "REDACTED",
+ "LaunchTime": {
+ "__class__": "datetime",
+ "year": 2018,
+ "month": 3,
+ "day": 20,
+ "hour": 15,
+ "minute": 35,
+ "second": 16,
+ "microsecond": 0
+ },
+ "Monitoring": {
+ "State": "disabled"
+ },
+ "Placement": {
+ "AvailabilityZone": "us-west-2b",
+ "GroupName": "",
+ "Tenancy": "default"
+ },
+ "PrivateDnsName": "ip-172-31-30-54.us-west-2.compute.internal",
+ "PrivateIpAddress": "172.31.30.54",
+ "ProductCodes": [],
+ "PublicDnsName": "ec2-34-209-91-146.us-west-2.compute.amazonaws.com",
+ "PublicIpAddress": "34.209.91.146",
+ "State": {
+ "Code": 16,
+ "Name": "running"
+ },
+ "StateTransitionReason": "",
+ "SubnetId": "subnet-05d5d863",
+ "VpcId": "vpc-02c00d7b",
+ "Architecture": "x86_64",
+ "BlockDeviceMappings": [],
+ "ClientToken": "",
+ "EbsOptimized": true,
+ "EnaSupport": true,
+ "Hypervisor": "xen",
+ "IamInstanceProfile": {
+ "Arn": "arn:aws:iam::REDACTED:instance-profile/jenkins_slave_profile",
+ "Id": "AIPAIJA6M2HUUH6EQTA44"
+ },
+ "NetworkInterfaces": [],
+ "RootDeviceName": "/dev/sda1",
+ "RootDeviceType": "ebs",
+ "SecurityGroups": [],
+ "SourceDestCheck": true,
+ "Tags": [
+ {
+ "Key": "Name",
+ "Value": "mxnetlinux-gpu12"
+ },
+ {
+ "Key": "label",
+ "Value": "mxnetlinux-gpu"
+ },
+ {
+ "Key": "AutoScaledSlave",
+ "Value": "True"
+ }
+ ],
+ "VirtualizationType": "hvm"
+ },
+ {
+ "AmiLaunchIndex": 0,
+ "ImageId": "ami-20e17d58",
+ "InstanceId": "i-000000000000004",
+ "InstanceType": "g3.8xlarge",
+ "KeyName": "REDACTED",
+ "LaunchTime": {
+ "__class__": "datetime",
+ "year": 2018,
+ "month": 3,
+ "day": 20,
+ "hour": 15,
+ "minute": 35,
+ "second": 16,
+ "microsecond": 0
+ },
+ "Monitoring": {
+ "State": "disabled"
+ },
+ "Placement": {
+ "AvailabilityZone": "us-west-2b",
+ "GroupName": "",
+ "Tenancy": "default"
+ },
+ "PrivateDnsName": "ip-172-31-30-54.us-west-2.compute.internal",
+ "PrivateIpAddress": "172.31.30.54",
+ "ProductCodes": [],
+ "PublicDnsName": "ec2-34-209-91-146.us-west-2.compute.amazonaws.com",
+ "PublicIpAddress": "34.209.91.146",
+ "State": {
+ "Code": 16,
+ "Name": "running"
+ },
+ "StateTransitionReason": "",
+ "SubnetId": "subnet-05d5d863",
+ "VpcId": "vpc-02c00d7b",
+ "Architecture": "x86_64",
+ "BlockDeviceMappings": [],
+ "ClientToken": "",
+ "EbsOptimized": true,
+ "EnaSupport": true,
+ "Hypervisor": "xen",
+ "IamInstanceProfile": {
+ "Arn": "arn:aws:iam::REDACTED:instance-profile/jenkins_slave_profile",
+ "Id": "AIPAIJA6M2HUUH6EQTA44"
+ },
+ "NetworkInterfaces": [],
+ "RootDeviceName": "/dev/sda1",
+ "RootDeviceType": "ebs",
+ "SecurityGroups": [],
+ "SourceDestCheck": true,
+ "Tags": [
+ {
+ "Key": "Name",
+ "Value": "mxnetlinux-cpu10"
+ },
+ {
+ "Key": "label",
+ "Value": "mxnetlinux-cpu"
+ },
+ {
+ "Key": "AutoScaledSlave",
+ "Value": "True"
+ }
+ ],
+ "VirtualizationType": "hvm"
+ },
+ {
+ "AmiLaunchIndex": 0,
+ "ImageId": "ami-20e17d58",
+ "InstanceId": "i-000000000000005",
+ "InstanceType": "g3.8xlarge",
+ "KeyName": "REDACTED",
+ "LaunchTime": {
+ "__class__": "datetime",
+ "year": 2018,
+ "month": 3,
+ "day": 20,
+ "hour": 15,
+ "minute": 35,
+ "second": 16,
+ "microsecond": 0
+ },
+ "Monitoring": {
+ "State": "disabled"
+ },
+ "Placement": {
+ "AvailabilityZone": "us-west-2b",
+ "GroupName": "",
+ "Tenancy": "default"
+ },
+ "PrivateDnsName": "ip-172-31-30-54.us-west-2.compute.internal",
+ "PrivateIpAddress": "172.31.30.54",
+ "ProductCodes": [],
+ "PublicDnsName": "ec2-34-209-91-146.us-west-2.compute.amazonaws.com",
+ "PublicIpAddress": "34.209.91.146",
+ "State": {
+ "Code": 16,
+ "Name": "running"
+ },
+ "StateTransitionReason": "",
+ "SubnetId": "subnet-05d5d863",
+ "VpcId": "vpc-02c00d7b",
+ "Architecture": "x86_64",
+ "BlockDeviceMappings": [],
+ "ClientToken": "",
+ "EbsOptimized": true,
+ "EnaSupport": true,
+ "Hypervisor": "xen",
+ "IamInstanceProfile": {
+ "Arn": "arn:aws:iam::REDACTED:instance-profile/jenkins_slave_profile",
+ "Id": "AIPAIJA6M2HUUH6EQTA44"
+ },
+ "NetworkInterfaces": [],
+ "RootDeviceName": "/dev/sda1",
+ "RootDeviceType": "ebs",
+ "SecurityGroups": [],
+ "SourceDestCheck": true,
+ "Tags": [
+ {
+ "Key": "Name",
+ "Value": "mxnetlinux-cpu11"
+ },
+ {
+ "Key": "label",
+ "Value": "mxnetlinux-cpu"
+ },
+ {
+ "Key": "AutoScaledSlave",
+ "Value": "True"
+ }
+ ],
+ "VirtualizationType": "hvm"
+ }
+ ],
+ "OwnerId": "REDACTED",
+ "ReservationId": "r-07430200188ec23e1"
+ }
+ ],
+ "ResponseMetadata": {
+ "RequestId": "5b0481d9-7ef4-45cf-8c58-7cf7efb9bd08",
+ "HTTPStatusCode": 200,
+ "HTTPHeaders": {
+ "content-type": "text/xml;charset=UTF-8",
+ "transfer-encoding": "chunked",
+ "vary": "Accept-Encoding",
+ "date": "Thu, 22 Mar 2018 23:38:12 GMT",
+ "server": "AmazonEC2"
+ },
+ "RetryAttempts": 0
+ }
+ }
+}
\ No newline at end of file
diff --git a/services/jenkins-autoscaling/lambda_mxnet_ci/tests/placebo/test_autoscaling/test_instance_uptime/ec2.DescribeInstances_1.json b/services/jenkins-autoscaling/lambda_mxnet_ci/tests/placebo/test_autoscaling/test_instance_uptime/ec2.DescribeInstances_1.json
new file mode 100644
index 0000000..f691316
--- /dev/null
+++ b/services/jenkins-autoscaling/lambda_mxnet_ci/tests/placebo/test_autoscaling/test_instance_uptime/ec2.DescribeInstances_1.json
@@ -0,0 +1,520 @@
+{
+ "status_code": 200,
+ "data": {
+ "Reservations": [
+ {
+ "Groups": [],
+ "Instances": [
+ {
+ "AmiLaunchIndex": 0,
+ "ImageId": "ami-07badfbfe163ec546",
+ "InstanceId": "i-000000000000001",
+ "InstanceType": "c5.18xlarge",
+ "KeyName": "key",
+ "LaunchTime": {
+ "__class__": "datetime",
+ "year": 2018,
+ "month": 4,
+ "day": 1,
+ "hour": 5,
+ "minute": 0,
+ "second": 0,
+ "microsecond": 0
+ },
+ "Monitoring": {
+ "State": "disabled"
+ },
+ "Placement": {
+ "AvailabilityZone": "us-west-2b",
+ "GroupName": "",
+ "Tenancy": "default"
+ },
+ "PrivateDnsName": "ip-172-31-28-108.us-west-2.compute.internal",
+ "PrivateIpAddress": "172.31.28.108",
+ "ProductCodes": [],
+ "PublicDnsName": "ec2-35-161-160-237.us-west-2.compute.amazonaws.com",
+ "PublicIpAddress": "35.161.160.237",
+ "State": {
+ "Code": 16,
+ "Name": "running"
+ },
+ "StateTransitionReason": "",
+ "SubnetId": "subnet-05d5d863",
+ "VpcId": "vpc-02c00d7b",
+ "Architecture": "x86_64",
+ "BlockDeviceMappings": [
+ {
+ "DeviceName": "/dev/sda1",
+ "Ebs": {
+ "AttachTime": {
+ "__class__": "datetime",
+ "year": 2018,
+ "month": 4,
+ "day": 30,
+ "hour": 4,
+ "minute": 5,
+ "second": 13,
+ "microsecond": 0
+ },
+ "DeleteOnTermination": true,
+ "Status": "attached",
+ "VolumeId": "vol-074269600fdddd8de"
+ }
+ }
+ ],
+ "ClientToken": "",
+ "EbsOptimized": false,
+ "EnaSupport": true,
+ "Hypervisor": "xen",
+ "IamInstanceProfile": {
+ "Arn": "arn:aws:iam::REDACTED:instance-profile/jenkins_slave_profile",
+ "Id": "AIPAIJA6M2HUUH6EQTA44"
+ },
+ "NetworkInterfaces": [
+ {
+ "Association": {
+ "IpOwnerId": "amazon",
+ "PublicDnsName": "ec2-35-161-160-237.us-west-2.compute.amazonaws.com",
+ "PublicIp": "35.161.160.237"
+ },
+ "Attachment": {
+ "AttachTime": {
+ "__class__": "datetime",
+ "year": 2018,
+ "month": 4,
+ "day": 30,
+ "hour": 4,
+ "minute": 5,
+ "second": 12,
+ "microsecond": 0
+ },
+ "AttachmentId": "eni-attach-06d3eff7ffeb36917",
+ "DeleteOnTermination": true,
+ "DeviceIndex": 0,
+ "Status": "attached"
+ },
+ "Description": "",
+ "Groups": [
+ {
+ "GroupName": "default",
+ "GroupId": "sg-5d83d421"
+ },
+ {
+ "GroupName": "REDACTED",
+ "GroupId": "REDACTED"
+ }
+ ],
+ "Ipv6Addresses": [],
+ "MacAddress": "02:80:49:f0:3a:a2",
+ "NetworkInterfaceId": "eni-0eae66c03e411513b",
+ "OwnerId": "REDACTED",
+ "PrivateDnsName": "ip-172-31-28-108.us-west-2.compute.internal",
+ "PrivateIpAddress": "172.31.28.108",
+ "PrivateIpAddresses": [
+ {
+ "Association": {
+ "IpOwnerId": "amazon",
+ "PublicDnsName": "ec2-35-161-160-237.us-west-2.compute.amazonaws.com",
+ "PublicIp": "35.161.160.237"
+ },
+ "Primary": true,
+ "PrivateDnsName": "ip-172-31-28-108.us-west-2.compute.internal",
+ "PrivateIpAddress": "172.31.28.108"
+ }
+ ],
+ "SourceDestCheck": true,
+ "Status": "in-use",
+ "SubnetId": "subnet-05d5d863",
+ "VpcId": "vpc-02c00d7b"
+ }
+ ],
+ "RootDeviceName": "/dev/sda1",
+ "RootDeviceType": "ebs",
+ "SecurityGroups": [
+ {
+ "GroupName": "default",
+ "GroupId": "sg-5d83d421"
+ },
+ {
+ "GroupName": "REDACTED",
+ "GroupId": "REDACTED"
+ }
+ ],
+ "SourceDestCheck": true,
+ "Tags": [
+ {
+ "Key": "label",
+ "Value": "mxnetlinux-cpu"
+ },
+ {
+ "Key": "aws:ec2launchtemplate:id",
+ "Value": "lt-06a15945813ad44f2"
+ },
+ {
+ "Key": "AutoScaledSlave",
+ "Value": "True"
+ },
+ {
+ "Key": "Name",
+ "Value": "node1"
+ },
+ {
+ "Key": "aws:ec2launchtemplate:version",
+ "Value": "2"
+ }
+ ],
+ "VirtualizationType": "hvm"
+ }
+ ],
+ "OwnerId": "REDACTED",
+ "ReservationId": "r-0cf1e4580eedf97d6"
+ },
+ {
+ "Groups": [],
+ "Instances": [
+ {
+ "AmiLaunchIndex": 0,
+ "ImageId": "ami-07badfbfe163ec546",
+ "InstanceId": "i-000000000000002",
+ "InstanceType": "c5.18xlarge",
+ "KeyName": "key",
+ "LaunchTime": {
+ "__class__": "datetime",
+ "year": 2018,
+ "month": 3,
+ "day": 1,
+ "hour": 5,
+ "minute": 10,
+ "second": 0,
+ "microsecond": 0
+ },
+ "Monitoring": {
+ "State": "disabled"
+ },
+ "Placement": {
+ "AvailabilityZone": "us-west-2b",
+ "GroupName": "",
+ "Tenancy": "default"
+ },
+ "PrivateDnsName": "ip-172-31-19-14.us-west-2.compute.internal",
+ "PrivateIpAddress": "172.31.19.14",
+ "ProductCodes": [],
+ "PublicDnsName": "ec2-54-189-192-122.us-west-2.compute.amazonaws.com",
+ "PublicIpAddress": "54.189.192.122",
+ "State": {
+ "Code": 16,
+ "Name": "running"
+ },
+ "StateTransitionReason": "",
+ "SubnetId": "subnet-05d5d863",
+ "VpcId": "vpc-02c00d7b",
+ "Architecture": "x86_64",
+ "BlockDeviceMappings": [
+ {
+ "DeviceName": "/dev/sda1",
+ "Ebs": {
+ "AttachTime": {
+ "__class__": "datetime",
+ "year": 2018,
+ "month": 4,
+ "day": 30,
+ "hour": 4,
+ "minute": 5,
+ "second": 16,
+ "microsecond": 0
+ },
+ "DeleteOnTermination": true,
+ "Status": "attached",
+ "VolumeId": "vol-0b453d50e16299c34"
+ }
+ }
+ ],
+ "ClientToken": "",
+ "EbsOptimized": false,
+ "EnaSupport": true,
+ "Hypervisor": "xen",
+ "IamInstanceProfile": {
+ "Arn": "arn:aws:iam::REDACTED:instance-profile/jenkins_slave_profile",
+ "Id": "AIPAIJA6M2HUUH6EQTA44"
+ },
+ "NetworkInterfaces": [
+ {
+ "Association": {
+ "IpOwnerId": "amazon",
+ "PublicDnsName": "ec2-54-189-192-122.us-west-2.compute.amazonaws.com",
+ "PublicIp": "54.189.192.122"
+ },
+ "Attachment": {
+ "AttachTime": {
+ "__class__": "datetime",
+ "year": 2018,
+ "month": 4,
+ "day": 30,
+ "hour": 4,
+ "minute": 5,
+ "second": 15,
+ "microsecond": 0
+ },
+ "AttachmentId": "eni-attach-0241d8aa1cdb9dcd3",
+ "DeleteOnTermination": true,
+ "DeviceIndex": 0,
+ "Status": "attached"
+ },
+ "Description": "",
+ "Groups": [
+ {
+ "GroupName": "default",
+ "GroupId": "sg-5d83d421"
+ },
+ {
+ "GroupName": "REDACTED",
+ "GroupId": "REDACTED"
+ }
+ ],
+ "Ipv6Addresses": [],
+ "MacAddress": "02:ec:57:07:4a:98",
+ "NetworkInterfaceId": "eni-0bf9e1b9da5c06b50",
+ "OwnerId": "REDACTED",
+ "PrivateDnsName": "ip-172-31-19-14.us-west-2.compute.internal",
+ "PrivateIpAddress": "172.31.19.14",
+ "PrivateIpAddresses": [
+ {
+ "Association": {
+ "IpOwnerId": "amazon",
+ "PublicDnsName": "ec2-54-189-192-122.us-west-2.compute.amazonaws.com",
+ "PublicIp": "54.189.192.122"
+ },
+ "Primary": true,
+ "PrivateDnsName": "ip-172-31-19-14.us-west-2.compute.internal",
+ "PrivateIpAddress": "172.31.19.14"
+ }
+ ],
+ "SourceDestCheck": true,
+ "Status": "in-use",
+ "SubnetId": "subnet-05d5d863",
+ "VpcId": "vpc-02c00d7b"
+ }
+ ],
+ "RootDeviceName": "/dev/sda1",
+ "RootDeviceType": "ebs",
+ "SecurityGroups": [
+ {
+ "GroupName": "default",
+ "GroupId": "sg-5d83d421"
+ },
+ {
+ "GroupName": "REDACTED",
+ "GroupId": "REDACTED"
+ }
+ ],
+ "SourceDestCheck": true,
+ "Tags": [
+ {
+ "Key": "AutoScaledSlave",
+ "Value": "True"
+ },
+ {
+ "Key": "label",
+ "Value": "mxnetlinux-cpu"
+ },
+ {
+ "Key": "aws:ec2launchtemplate:id",
+ "Value": "lt-06a15945813ad44f2"
+ },
+ {
+ "Key": "Name",
+ "Value": "node2"
+ },
+ {
+ "Key": "aws:ec2launchtemplate:version",
+ "Value": "2"
+ }
+ ],
+ "VirtualizationType": "hvm"
+ }
+ ],
+ "OwnerId": "REDACTED",
+ "ReservationId": "r-088bba83e9c4e7d48"
+ },
+ {
+ "Groups": [],
+ "Instances": [
+ {
+ "AmiLaunchIndex": 0,
+ "ImageId": "ami-07badfbfe163ec546",
+ "InstanceId": "i-000000000000003",
+ "InstanceType": "c5.18xlarge",
+ "KeyName": "key",
+ "LaunchTime": {
+ "__class__": "datetime",
+ "year": 2018,
+ "month": 3,
+ "day": 5,
+ "hour": 15,
+ "minute": 5,
+ "second": 0,
+ "microsecond": 0
+ },
+ "Monitoring": {
+ "State": "disabled"
+ },
+ "Placement": {
+ "AvailabilityZone": "us-west-2b",
+ "GroupName": "",
+ "Tenancy": "default"
+ },
+ "PrivateDnsName": "ip-172-31-25-219.us-west-2.compute.internal",
+ "PrivateIpAddress": "172.31.25.219",
+ "ProductCodes": [],
+ "PublicDnsName": "ec2-54-200-177-135.us-west-2.compute.amazonaws.com",
+ "PublicIpAddress": "54.200.177.135",
+ "State": {
+ "Code": 16,
+ "Name": "running"
+ },
+ "StateTransitionReason": "",
+ "SubnetId": "subnet-05d5d863",
+ "VpcId": "vpc-02c00d7b",
+ "Architecture": "x86_64",
+ "BlockDeviceMappings": [
+ {
+ "DeviceName": "/dev/sda1",
+ "Ebs": {
+ "AttachTime": {
+ "__class__": "datetime",
+ "year": 2018,
+ "month": 4,
+ "day": 30,
+ "hour": 4,
+ "minute": 5,
+ "second": 15,
+ "microsecond": 0
+ },
+ "DeleteOnTermination": true,
+ "Status": "attached",
+ "VolumeId": "vol-0276696f36a10b8c1"
+ }
+ }
+ ],
+ "ClientToken": "",
+ "EbsOptimized": false,
+ "EnaSupport": true,
+ "Hypervisor": "xen",
+ "IamInstanceProfile": {
+ "Arn": "arn:aws:iam::REDACTED:instance-profile/jenkins_slave_profile",
+ "Id": "AIPAIJA6M2HUUH6EQTA44"
+ },
+ "NetworkInterfaces": [
+ {
+ "Association": {
+ "IpOwnerId": "amazon",
+ "PublicDnsName": "ec2-54-200-177-135.us-west-2.compute.amazonaws.com",
+ "PublicIp": "54.200.177.135"
+ },
+ "Attachment": {
+ "AttachTime": {
+ "__class__": "datetime",
+ "year": 2018,
+ "month": 4,
+ "day": 30,
+ "hour": 4,
+ "minute": 5,
+ "second": 14,
+ "microsecond": 0
+ },
+ "AttachmentId": "eni-attach-0f3b113940741e8a2",
+ "DeleteOnTermination": true,
+ "DeviceIndex": 0,
+ "Status": "attached"
+ },
+ "Description": "",
+ "Groups": [
+ {
+ "GroupName": "default",
+ "GroupId": "sg-5d83d421"
+ },
+ {
+ "GroupName": "REDACTED",
+ "GroupId": "REDACTED"
+ }
+ ],
+ "Ipv6Addresses": [],
+ "MacAddress": "02:a0:e6:23:0c:12",
+ "NetworkInterfaceId": "eni-053455704c4e28aa2",
+ "OwnerId": "REDACTED",
+ "PrivateDnsName": "ip-172-31-25-219.us-west-2.compute.internal",
+ "PrivateIpAddress": "172.31.25.219",
+ "PrivateIpAddresses": [
+ {
+ "Association": {
+ "IpOwnerId": "amazon",
+ "PublicDnsName": "ec2-54-200-177-135.us-west-2.compute.amazonaws.com",
+ "PublicIp": "54.200.177.135"
+ },
+ "Primary": true,
+ "PrivateDnsName": "ip-172-31-25-219.us-west-2.compute.internal",
+ "PrivateIpAddress": "172.31.25.219"
+ }
+ ],
+ "SourceDestCheck": true,
+ "Status": "in-use",
+ "SubnetId": "subnet-05d5d863",
+ "VpcId": "vpc-02c00d7b"
+ }
+ ],
+ "RootDeviceName": "/dev/sda1",
+ "RootDeviceType": "ebs",
+ "SecurityGroups": [
+ {
+ "GroupName": "default",
+ "GroupId": "sg-5d83d421"
+ },
+ {
+ "GroupName": "REDACTED",
+ "GroupId": "REDACTED"
+ }
+ ],
+ "SourceDestCheck": true,
+ "Tags": [
+ {
+ "Key": "Name",
+ "Value": "node3"
+ },
+ {
+ "Key": "AutoScaledSlave",
+ "Value": "True"
+ },
+ {
+ "Key": "label",
+ "Value": "mxnetlinux-cpu"
+ },
+ {
+ "Key": "aws:ec2launchtemplate:id",
+ "Value": "lt-06a15945813ad44f2"
+ },
+ {
+ "Key": "aws:ec2launchtemplate:version",
+ "Value": "2"
+ }
+ ],
+ "VirtualizationType": "hvm"
+ }
+ ],
+ "OwnerId": "REDACTED",
+ "ReservationId": "r-080933a815321b6ef"
+ }
+ ],
+ "ResponseMetadata": {
+ "RequestId": "e2655f9a-6f98-4cb8-96ba-b705939cdc99",
+ "HTTPStatusCode": 200,
+ "HTTPHeaders": {
+ "content-type": "text/xml;charset=UTF-8",
+ "transfer-encoding": "chunked",
+ "vary": "Accept-Encoding",
+ "date": "Mon, 30 Apr 2018 04:11:51 GMT",
+ "server": "AmazonEC2"
+ },
+ "RetryAttempts": 0
+ }
+ }
+}
\ No newline at end of file
diff --git a/services/jenkins-autoscaling/lambda_mxnet_ci/tests/placebo/test_autoscaling/test_terminate_ec2_instances/ec2.DescribeInstances_1.json b/services/jenkins-autoscaling/lambda_mxnet_ci/tests/placebo/test_autoscaling/test_terminate_ec2_instances/ec2.DescribeInstances_1.json
new file mode 100644
index 0000000..fc6a667
--- /dev/null
+++ b/services/jenkins-autoscaling/lambda_mxnet_ci/tests/placebo/test_autoscaling/test_terminate_ec2_instances/ec2.DescribeInstances_1.json
@@ -0,0 +1,294 @@
+{
+ "status_code": 200,
+ "data": {
+ "Reservations": [
+ {
+ "Groups": [],
+ "Instances": [
+ {
+ "AmiLaunchIndex": 0,
+ "ImageId": "ami-20e17d58",
+ "InstanceId": "i-000000000000001",
+ "InstanceType": "g3.8xlarge",
+ "KeyName": "REDACTED",
+ "LaunchTime": {
+ "__class__": "datetime",
+ "year": 2018,
+ "month": 3,
+ "day": 20,
+ "hour": 15,
+ "minute": 35,
+ "second": 16,
+ "microsecond": 0
+ },
+ "Monitoring": {
+ "State": "disabled"
+ },
+ "Placement": {
+ "AvailabilityZone": "us-west-2b",
+ "GroupName": "",
+ "Tenancy": "default"
+ },
+ "PrivateDnsName": "ip-172-31-30-54.us-west-2.compute.internal",
+ "PrivateIpAddress": "172.31.30.54",
+ "ProductCodes": [],
+ "PublicDnsName": "ec2-34-209-91-146.us-west-2.compute.amazonaws.com",
+ "PublicIpAddress": "34.209.91.146",
+ "State": {
+ "Code": 16,
+ "Name": "running"
+ },
+ "StateTransitionReason": "",
+ "SubnetId": "subnet-05d5d863",
+ "VpcId": "vpc-02c00d7b",
+ "Architecture": "x86_64",
+ "BlockDeviceMappings": [],
+ "ClientToken": "",
+ "EbsOptimized": true,
+ "EnaSupport": true,
+ "Hypervisor": "xen",
+ "IamInstanceProfile": {
+ "Arn": "arn:aws:iam::REDACTED:instance-profile/jenkins_slave_profile",
+ "Id": "AIPAIJA6M2HUUH6EQTA44"
+ },
+ "NetworkInterfaces": [],
+ "RootDeviceName": "/dev/sda1",
+ "RootDeviceType": "ebs",
+ "SecurityGroups": [],
+ "SourceDestCheck": true,
+ "Tags": [
+ {
+ "Key": "Name",
+ "Value": "terminate1"
+ },
+ {
+ "Key": "label",
+ "Value": "mxnetlinux-gpu"
+ },
+ {
+ "Key": "AutoScaledSlave",
+ "Value": "True"
+ }
+ ],
+ "VirtualizationType": "hvm"
+ },
+ {
+ "AmiLaunchIndex": 0,
+ "ImageId": "ami-20e17d58",
+ "InstanceId": "i-000000000000002",
+ "InstanceType": "g3.8xlarge",
+ "KeyName": "REDACTED",
+ "LaunchTime": {
+ "__class__": "datetime",
+ "year": 2018,
+ "month": 3,
+ "day": 20,
+ "hour": 15,
+ "minute": 35,
+ "second": 16,
+ "microsecond": 0
+ },
+ "Monitoring": {
+ "State": "disabled"
+ },
+ "Placement": {
+ "AvailabilityZone": "us-west-2b",
+ "GroupName": "",
+ "Tenancy": "default"
+ },
+ "PrivateDnsName": "ip-172-31-30-54.us-west-2.compute.internal",
+ "PrivateIpAddress": "172.31.30.54",
+ "ProductCodes": [],
+ "PublicDnsName": "ec2-34-209-91-146.us-west-2.compute.amazonaws.com",
+ "PublicIpAddress": "34.209.91.146",
+ "State": {
+ "Code": 16,
+ "Name": "running"
+ },
+ "StateTransitionReason": "",
+ "SubnetId": "subnet-05d5d863",
+ "VpcId": "vpc-02c00d7b",
+ "Architecture": "x86_64",
+ "BlockDeviceMappings": [],
+ "ClientToken": "",
+ "EbsOptimized": true,
+ "EnaSupport": true,
+ "Hypervisor": "xen",
+ "IamInstanceProfile": {
+ "Arn": "arn:aws:iam::REDACTED:instance-profile/jenkins_slave_profile",
+ "Id": "AIPAIJA6M2HUUH6EQTA44"
+ },
+ "NetworkInterfaces": [],
+ "RootDeviceName": "/dev/sda1",
+ "RootDeviceType": "ebs",
+ "SecurityGroups": [],
+ "SourceDestCheck": true,
+ "Tags": [
+ {
+ "Key": "Name",
+ "Value": "terminate2"
+ },
+ {
+ "Key": "label",
+ "Value": "mxnetlinux-gpu"
+ },
+ {
+ "Key": "AutoScaledSlave",
+ "Value": "True"
+ }
+ ],
+ "VirtualizationType": "hvm"
+ },
+ {
+ "AmiLaunchIndex": 0,
+ "ImageId": "ami-20e17d58",
+ "InstanceId": "i-000000000000003",
+ "InstanceType": "g3.8xlarge",
+ "KeyName": "REDACTED",
+ "LaunchTime": {
+ "__class__": "datetime",
+ "year": 2018,
+ "month": 3,
+ "day": 20,
+ "hour": 15,
+ "minute": 35,
+ "second": 16,
+ "microsecond": 0
+ },
+ "Monitoring": {
+ "State": "disabled"
+ },
+ "Placement": {
+ "AvailabilityZone": "us-west-2b",
+ "GroupName": "",
+ "Tenancy": "default"
+ },
+ "PrivateDnsName": "ip-172-31-30-54.us-west-2.compute.internal",
+ "PrivateIpAddress": "172.31.30.54",
+ "ProductCodes": [],
+ "PublicDnsName": "ec2-34-209-91-146.us-west-2.compute.amazonaws.com",
+ "PublicIpAddress": "34.209.91.146",
+ "State": {
+ "Code": 16,
+ "Name": "running"
+ },
+ "StateTransitionReason": "",
+ "SubnetId": "subnet-05d5d863",
+ "VpcId": "vpc-02c00d7b",
+ "Architecture": "x86_64",
+ "BlockDeviceMappings": [],
+ "ClientToken": "",
+ "EbsOptimized": true,
+ "EnaSupport": true,
+ "Hypervisor": "xen",
+ "IamInstanceProfile": {
+ "Arn": "arn:aws:iam::REDACTED:instance-profile/jenkins_slave_profile",
+ "Id": "AIPAIJA6M2HUUH6EQTA44"
+ },
+ "NetworkInterfaces": [],
+ "RootDeviceName": "/dev/sda1",
+ "RootDeviceType": "ebs",
+ "SecurityGroups": [],
+ "SourceDestCheck": true,
+ "Tags": [
+ {
+ "Key": "Name",
+ "Value": "terminate3"
+ },
+ {
+ "Key": "label",
+ "Value": "mxnetlinux-gpu"
+ },
+ {
+ "Key": "AutoScaledSlave",
+ "Value": "True"
+ }
+ ],
+ "VirtualizationType": "hvm"
+ },
+ {
+ "AmiLaunchIndex": 0,
+ "ImageId": "ami-20e17d58",
+ "InstanceId": "i-000000000000004",
+ "InstanceType": "g3.8xlarge",
+ "KeyName": "REDACTED",
+ "LaunchTime": {
+ "__class__": "datetime",
+ "year": 2018,
+ "month": 3,
+ "day": 20,
+ "hour": 15,
+ "minute": 35,
+ "second": 16,
+ "microsecond": 0
+ },
+ "Monitoring": {
+ "State": "disabled"
+ },
+ "Placement": {
+ "AvailabilityZone": "us-west-2b",
+ "GroupName": "",
+ "Tenancy": "default"
+ },
+ "PrivateDnsName": "ip-172-31-30-54.us-west-2.compute.internal",
+ "PrivateIpAddress": "172.31.30.54",
+ "ProductCodes": [],
+ "PublicDnsName": "ec2-34-209-91-146.us-west-2.compute.amazonaws.com",
+ "PublicIpAddress": "34.209.91.146",
+ "State": {
+ "Code": 16,
+ "Name": "running"
+ },
+ "StateTransitionReason": "",
+ "SubnetId": "subnet-05d5d863",
+ "VpcId": "vpc-02c00d7b",
+ "Architecture": "x86_64",
+ "BlockDeviceMappings": [],
+ "ClientToken": "",
+ "EbsOptimized": true,
+ "EnaSupport": true,
+ "Hypervisor": "xen",
+ "IamInstanceProfile": {
+ "Arn": "arn:aws:iam::REDACTED:instance-profile/jenkins_slave_profile",
+ "Id": "AIPAIJA6M2HUUH6EQTA44"
+ },
+ "NetworkInterfaces": [],
+ "RootDeviceName": "/dev/sda1",
+ "RootDeviceType": "ebs",
+ "SecurityGroups": [],
+ "SourceDestCheck": true,
+ "Tags": [
+ {
+ "Key": "Name",
+ "Value": "terminate4"
+ },
+ {
+ "Key": "label",
+ "Value": "mxnetlinux-gpu"
+ },
+ {
+ "Key": "AutoScaledSlave",
+ "Value": "True"
+ }
+ ],
+ "VirtualizationType": "hvm"
+ }
+ ],
+ "OwnerId": "REDACTED",
+ "ReservationId": "r-07430200188ec23e1"
+ }
+ ],
+ "ResponseMetadata": {
+ "RequestId": "dfd97b82-c09d-455e-8bc4-cc2eb366f7ba",
+ "HTTPStatusCode": 200,
+ "HTTPHeaders": {
+ "content-type": "text/xml;charset=UTF-8",
+ "transfer-encoding": "chunked",
+ "vary": "Accept-Encoding",
+ "date": "Sun, 22 Apr 2018 23:08:29 GMT",
+ "server": "AmazonEC2"
+ },
+ "RetryAttempts": 0
+ }
+ }
+}
\ No newline at end of file
diff --git a/services/jenkins-autoscaling/lambda_mxnet_ci/tests/placebo/test_autoscaling/test_terminate_ec2_instances/ec2.DescribeInstances_2.json b/services/jenkins-autoscaling/lambda_mxnet_ci/tests/placebo/test_autoscaling/test_terminate_ec2_instances/ec2.DescribeInstances_2.json
new file mode 100644
index 0000000..fc6a667
--- /dev/null
+++ b/services/jenkins-autoscaling/lambda_mxnet_ci/tests/placebo/test_autoscaling/test_terminate_ec2_instances/ec2.DescribeInstances_2.json
@@ -0,0 +1,294 @@
+{
+ "status_code": 200,
+ "data": {
+ "Reservations": [
+ {
+ "Groups": [],
+ "Instances": [
+ {
+ "AmiLaunchIndex": 0,
+ "ImageId": "ami-20e17d58",
+ "InstanceId": "i-000000000000001",
+ "InstanceType": "g3.8xlarge",
+ "KeyName": "REDACTED",
+ "LaunchTime": {
+ "__class__": "datetime",
+ "year": 2018,
+ "month": 3,
+ "day": 20,
+ "hour": 15,
+ "minute": 35,
+ "second": 16,
+ "microsecond": 0
+ },
+ "Monitoring": {
+ "State": "disabled"
+ },
+ "Placement": {
+ "AvailabilityZone": "us-west-2b",
+ "GroupName": "",
+ "Tenancy": "default"
+ },
+ "PrivateDnsName": "ip-172-31-30-54.us-west-2.compute.internal",
+ "PrivateIpAddress": "172.31.30.54",
+ "ProductCodes": [],
+ "PublicDnsName": "ec2-34-209-91-146.us-west-2.compute.amazonaws.com",
+ "PublicIpAddress": "34.209.91.146",
+ "State": {
+ "Code": 16,
+ "Name": "running"
+ },
+ "StateTransitionReason": "",
+ "SubnetId": "subnet-05d5d863",
+ "VpcId": "vpc-02c00d7b",
+ "Architecture": "x86_64",
+ "BlockDeviceMappings": [],
+ "ClientToken": "",
+ "EbsOptimized": true,
+ "EnaSupport": true,
+ "Hypervisor": "xen",
+ "IamInstanceProfile": {
+ "Arn": "arn:aws:iam::REDACTED:instance-profile/jenkins_slave_profile",
+ "Id": "AIPAIJA6M2HUUH6EQTA44"
+ },
+ "NetworkInterfaces": [],
+ "RootDeviceName": "/dev/sda1",
+ "RootDeviceType": "ebs",
+ "SecurityGroups": [],
+ "SourceDestCheck": true,
+ "Tags": [
+ {
+ "Key": "Name",
+ "Value": "terminate1"
+ },
+ {
+ "Key": "label",
+ "Value": "mxnetlinux-gpu"
+ },
+ {
+ "Key": "AutoScaledSlave",
+ "Value": "True"
+ }
+ ],
+ "VirtualizationType": "hvm"
+ },
+ {
+ "AmiLaunchIndex": 0,
+ "ImageId": "ami-20e17d58",
+ "InstanceId": "i-000000000000002",
+ "InstanceType": "g3.8xlarge",
+ "KeyName": "REDACTED",
+ "LaunchTime": {
+ "__class__": "datetime",
+ "year": 2018,
+ "month": 3,
+ "day": 20,
+ "hour": 15,
+ "minute": 35,
+ "second": 16,
+ "microsecond": 0
+ },
+ "Monitoring": {
+ "State": "disabled"
+ },
+ "Placement": {
+ "AvailabilityZone": "us-west-2b",
+ "GroupName": "",
+ "Tenancy": "default"
+ },
+ "PrivateDnsName": "ip-172-31-30-54.us-west-2.compute.internal",
+ "PrivateIpAddress": "172.31.30.54",
+ "ProductCodes": [],
+ "PublicDnsName": "ec2-34-209-91-146.us-west-2.compute.amazonaws.com",
+ "PublicIpAddress": "34.209.91.146",
+ "State": {
+ "Code": 16,
+ "Name": "running"
+ },
+ "StateTransitionReason": "",
+ "SubnetId": "subnet-05d5d863",
+ "VpcId": "vpc-02c00d7b",
+ "Architecture": "x86_64",
+ "BlockDeviceMappings": [],
+ "ClientToken": "",
+ "EbsOptimized": true,
+ "EnaSupport": true,
+ "Hypervisor": "xen",
+ "IamInstanceProfile": {
+ "Arn": "arn:aws:iam::REDACTED:instance-profile/jenkins_slave_profile",
+ "Id": "AIPAIJA6M2HUUH6EQTA44"
+ },
+ "NetworkInterfaces": [],
+ "RootDeviceName": "/dev/sda1",
+ "RootDeviceType": "ebs",
+ "SecurityGroups": [],
+ "SourceDestCheck": true,
+ "Tags": [
+ {
+ "Key": "Name",
+ "Value": "terminate2"
+ },
+ {
+ "Key": "label",
+ "Value": "mxnetlinux-gpu"
+ },
+ {
+ "Key": "AutoScaledSlave",
+ "Value": "True"
+ }
+ ],
+ "VirtualizationType": "hvm"
+ },
+ {
+ "AmiLaunchIndex": 0,
+ "ImageId": "ami-20e17d58",
+ "InstanceId": "i-000000000000003",
+ "InstanceType": "g3.8xlarge",
+ "KeyName": "REDACTED",
+ "LaunchTime": {
+ "__class__": "datetime",
+ "year": 2018,
+ "month": 3,
+ "day": 20,
+ "hour": 15,
+ "minute": 35,
+ "second": 16,
+ "microsecond": 0
+ },
+ "Monitoring": {
+ "State": "disabled"
+ },
+ "Placement": {
+ "AvailabilityZone": "us-west-2b",
+ "GroupName": "",
+ "Tenancy": "default"
+ },
+ "PrivateDnsName": "ip-172-31-30-54.us-west-2.compute.internal",
+ "PrivateIpAddress": "172.31.30.54",
+ "ProductCodes": [],
+ "PublicDnsName": "ec2-34-209-91-146.us-west-2.compute.amazonaws.com",
+ "PublicIpAddress": "34.209.91.146",
+ "State": {
+ "Code": 16,
+ "Name": "running"
+ },
+ "StateTransitionReason": "",
+ "SubnetId": "subnet-05d5d863",
+ "VpcId": "vpc-02c00d7b",
+ "Architecture": "x86_64",
+ "BlockDeviceMappings": [],
+ "ClientToken": "",
+ "EbsOptimized": true,
+ "EnaSupport": true,
+ "Hypervisor": "xen",
+ "IamInstanceProfile": {
+ "Arn": "arn:aws:iam::REDACTED:instance-profile/jenkins_slave_profile",
+ "Id": "AIPAIJA6M2HUUH6EQTA44"
+ },
+ "NetworkInterfaces": [],
+ "RootDeviceName": "/dev/sda1",
+ "RootDeviceType": "ebs",
+ "SecurityGroups": [],
+ "SourceDestCheck": true,
+ "Tags": [
+ {
+ "Key": "Name",
+ "Value": "terminate3"
+ },
+ {
+ "Key": "label",
+ "Value": "mxnetlinux-gpu"
+ },
+ {
+ "Key": "AutoScaledSlave",
+ "Value": "True"
+ }
+ ],
+ "VirtualizationType": "hvm"
+ },
+ {
+ "AmiLaunchIndex": 0,
+ "ImageId": "ami-20e17d58",
+ "InstanceId": "i-000000000000004",
+ "InstanceType": "g3.8xlarge",
+ "KeyName": "REDACTED",
+ "LaunchTime": {
+ "__class__": "datetime",
+ "year": 2018,
+ "month": 3,
+ "day": 20,
+ "hour": 15,
+ "minute": 35,
+ "second": 16,
+ "microsecond": 0
+ },
+ "Monitoring": {
+ "State": "disabled"
+ },
+ "Placement": {
+ "AvailabilityZone": "us-west-2b",
+ "GroupName": "",
+ "Tenancy": "default"
+ },
+ "PrivateDnsName": "ip-172-31-30-54.us-west-2.compute.internal",
+ "PrivateIpAddress": "172.31.30.54",
+ "ProductCodes": [],
+ "PublicDnsName": "ec2-34-209-91-146.us-west-2.compute.amazonaws.com",
+ "PublicIpAddress": "34.209.91.146",
+ "State": {
+ "Code": 16,
+ "Name": "running"
+ },
+ "StateTransitionReason": "",
+ "SubnetId": "subnet-05d5d863",
+ "VpcId": "vpc-02c00d7b",
+ "Architecture": "x86_64",
+ "BlockDeviceMappings": [],
+ "ClientToken": "",
+ "EbsOptimized": true,
+ "EnaSupport": true,
+ "Hypervisor": "xen",
+ "IamInstanceProfile": {
+ "Arn": "arn:aws:iam::REDACTED:instance-profile/jenkins_slave_profile",
+ "Id": "AIPAIJA6M2HUUH6EQTA44"
+ },
+ "NetworkInterfaces": [],
+ "RootDeviceName": "/dev/sda1",
+ "RootDeviceType": "ebs",
+ "SecurityGroups": [],
+ "SourceDestCheck": true,
+ "Tags": [
+ {
+ "Key": "Name",
+ "Value": "terminate4"
+ },
+ {
+ "Key": "label",
+ "Value": "mxnetlinux-gpu"
+ },
+ {
+ "Key": "AutoScaledSlave",
+ "Value": "True"
+ }
+ ],
+ "VirtualizationType": "hvm"
+ }
+ ],
+ "OwnerId": "REDACTED",
+ "ReservationId": "r-07430200188ec23e1"
+ }
+ ],
+ "ResponseMetadata": {
+ "RequestId": "dfd97b82-c09d-455e-8bc4-cc2eb366f7ba",
+ "HTTPStatusCode": 200,
+ "HTTPHeaders": {
+ "content-type": "text/xml;charset=UTF-8",
+ "transfer-encoding": "chunked",
+ "vary": "Accept-Encoding",
+ "date": "Sun, 22 Apr 2018 23:08:29 GMT",
+ "server": "AmazonEC2"
+ },
+ "RetryAttempts": 0
+ }
+ }
+}
\ No newline at end of file
diff --git a/services/jenkins-autoscaling/lambda_mxnet_ci/tests/placebo/test_autoscaling/test_terminate_ec2_instances/ec2.TerminateInstances_1.json b/services/jenkins-autoscaling/lambda_mxnet_ci/tests/placebo/test_autoscaling/test_terminate_ec2_instances/ec2.TerminateInstances_1.json
new file mode 100644
index 0000000..b76921b
--- /dev/null
+++ b/services/jenkins-autoscaling/lambda_mxnet_ci/tests/placebo/test_autoscaling/test_terminate_ec2_instances/ec2.TerminateInstances_1.json
@@ -0,0 +1,63 @@
+{
+ "status_code": 200,
+ "data": {
+ "TerminatingInstances": [
+ {
+ "CurrentState": {
+ "Code": 32,
+ "Name": "shutting-down"
+ },
+ "InstanceId": "i-000000000000001",
+ "PreviousState": {
+ "Code": 32,
+ "Name": "shutting-down"
+ }
+ },
+ {
+ "CurrentState": {
+ "Code": 32,
+ "Name": "shutting-down"
+ },
+ "InstanceId": "i-000000000000002",
+ "PreviousState": {
+ "Code": 32,
+ "Name": "shutting-down"
+ }
+ },
+ {
+ "CurrentState": {
+ "Code": 32,
+ "Name": "shutting-down"
+ },
+ "InstanceId": "i-000000000000003",
+ "PreviousState": {
+ "Code": 32,
+ "Name": "shutting-down"
+ }
+ },
+ {
+ "CurrentState": {
+ "Code": 32,
+ "Name": "shutting-down"
+ },
+ "InstanceId": "i-000000000000004",
+ "PreviousState": {
+ "Code": 32,
+ "Name": "shutting-down"
+ }
+ }
+ ],
+ "ResponseMetadata": {
+ "RequestId": "0972567a-f24e-4dee-913c-0839e3ec8b36",
+ "HTTPStatusCode": 200,
+ "HTTPHeaders": {
+ "content-type": "text/xml;charset=UTF-8",
+ "transfer-encoding": "chunked",
+ "vary": "Accept-Encoding",
+ "date": "Sun, 22 Apr 2018 23:08:30 GMT",
+ "server": "AmazonEC2"
+ },
+ "RetryAttempts": 0
+ }
+ }
+}
\ No newline at end of file
diff --git a/services/jenkins-autoscaling/lambda_mxnet_ci/tests/test_autoscaling.py b/services/jenkins-autoscaling/lambda_mxnet_ci/tests/test_autoscaling.py
new file mode 100644
index 0000000..77d119a
--- /dev/null
+++ b/services/jenkins-autoscaling/lambda_mxnet_ci/tests/test_autoscaling.py
@@ -0,0 +1,1258 @@
+#!/usr/bin/env python3
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# -*- coding: utf-8 -*-
+
+# pylint: disable=missing-docstring
+import datetime
+import logging
+import os
+import pprint
+import random
+import string
+import threading
+import time
+import unittest
+import unittest.mock
+from collections import defaultdict
+
+import math
+from typing import Dict, Any
+
+import boto3
+from botocore.exceptions import ClientError
+import jenkinsapi
+import placebo
+
+# import lambda_mxnet_ci.autoscaling.handler as autoscaling
+import autoscaling.handler as autoscaling
+
+EC2_PLACE_ACCOUNT_NAME = 'mxnet-ci-dev'
+
+QUEUE_WHY_FORMULATIONS = [
+ 'There are no nodes with the label ‘{}’',
+ 'Waiting for next available executor on {}',
+ 'All nodes of label ‘{}’ are offline',
+ 'Waiting for next available executor on {}',
+ 'Waiting for next available executor on ‘{}’',
+ 'mxnetlinux-cpu_0123456 doesn’t have label {}; node2 doesn’t have label blabla; node3 is offline',
+ '{} is offline'
+]
+
+# Thread locks have to be at this level because of joblib/pickle
+MOCK_FILTER_NON_IDLE_OFFLINE_NODES_THREAD_LOCK = threading.Lock()
+MOCK_CONVERT_JENKINS_NODES_THREAD_LOCK = threading.Lock()
+MOCK_JENKINS_POST_CONFIRM_THREAD_LOCK = threading.Lock()
+
+
+class MockJenkinsServer(object):
+ def __init__(self, baseurl):
+ self.requester = self
+ self.baseurl = baseurl
+ self.nb_post_called = 0
+
+ # pylint: disable=unused-argument
+ def post_and_confirm_status(self, url, params=None, data=None, files=None, headers=None,
+ valid=None, allow_redirects=True):
+ with MOCK_JENKINS_POST_CONFIRM_THREAD_LOCK:
+ self.nb_post_called += 1
+
+ # TODO: save data to validate it
+
+
+class TestMetrics(unittest.TestCase):
+ @staticmethod
+ def _util_create_node(display_name, assigned_labels, nb_executors, is_offline=False, temporarily_offline=False,
+ is_idle=False, architecture='Linux (amd64)',
+ offline_cause_class='hudson.slaves.OfflineCause$UserCause', offline_reason=''):
+ assigned_labels.insert(0, display_name) # A node always has its own name as well as the labels as labels
+
+ node = dict()
+ node['displayName'] = display_name
+ node['assignedLabels'] = [{'name': label} for label in assigned_labels]
+ node['offline'] = is_offline
+ node['temporarilyOffline'] = temporarily_offline
+ node['offlineCause'] = {'_class': offline_cause_class}
+ node['offlineCauseReason'] = offline_reason
+ node['monitorData'] = dict()
+ node['monitorData']['hudson.node_monitors.ArchitectureMonitor'] = architecture
+ node['idle'] = is_idle
+ if nb_executors:
+ executors = list()
+ for _ in range(0, nb_executors):
+ executors.append(None)
+ node['executors'] = executors
+
+ return node
+
+ @staticmethod
+ def _util_create_queue_item(reason: str, in_queue_duration_sec: int) -> Dict[str, Any]:
+ queue_item = dict()
+ queue_item['why'] = reason
+ queue_item['id'] = 0
+ queue_item['inQueueSince'] = 1000 * (time.time() - in_queue_duration_sec) # Stored in milliseconds
+ return queue_item
+
+ @staticmethod
+ def _util_get_placebo_session(current_file, test_name, record=False):
+ if record:
+ session = boto3.Session(profile_name=EC2_PLACE_ACCOUNT_NAME)
+ else:
+ session = boto3.Session()
+ placebo_dir = os.path.join(os.path.dirname(current_file), 'placebo',
+ os.path.splitext(os.path.basename(current_file))[0],
+ test_name)
+ os.makedirs(name=placebo_dir, exist_ok=True)
+
+ pill = placebo.attach(session, data_path=placebo_dir)
+ if record:
+ pill.record()
+ else:
+ pill.playback()
+
+ return session
+
+ def setUp(self):
+ # Make sure to properly override all functions that are creating external connections or would retrieve
+ # real credentials
+ get_jenkins_credentials_mock_method = unittest.mock.MagicMock()
+ get_jenkins_credentials_mock_method.return_value = {
+ 'github_username': 'github_username',
+ 'github_token': 'github_token',
+ 'jenkins_url': 'jenkins_url',
+ 'jenkins_priv_url': 'jenkins_priv_url'
+ }
+ autoscaling._get_jenkins_credentials = get_jenkins_credentials_mock_method
+ autoscaling._get_aws_session = unittest.mock.MagicMock()
+ autoscaling._get_jenkins_handle = unittest.mock.MagicMock()
+ os.environ['LAUNCH_TEMPLATES'] = \
+ '{"mxnetlinux-cpu":{"id":"lt-06a15945813ad44f2","version":"5"},' \
+ '"restricted-mxnetlinux-cpu":{"id":"lt-06a15945813ad44f2","version":"5"},' \
+ '"mxnetlinux-gpu":{"id":"lt-0c22f238c0edb58ab","version":"8"},' \
+ '"mxnetlinux-gpu-p3":{"id":"lt-00c83ee5d7aeaf4ab","version":"4"},' \
+ '"restricted-mxnetlinux-gpu-p3":{"id":"lt-0f893d7f3f2660c1c","version":"2"},' \
+ '"mxnetlinux-gpu-p3-8xlarge":{"id":"lt-00c83ee5d7aeaf4ab","version":"4"},' \
+ '"mxnetwindows-cpu":{"id":"lt-09dff2fff6b5586f0","version":"4"},' \
+ '"mxnetwindows-gpu":{"id":"lt-0ce229129d0d3be27","version":"6"},' \
+ '"restricted-mxnetlinux-gpu":{"id":"lt-0c22f238c0edb58ab","version":"1"} }'
+ os.environ['EXECUTORS_PER_LABEL'] = \
+ '{"mxnetlinux-cpu":3,"restricted-mxnetlinux-cpu":3,"mxnetlinux-gpu":1,"mxnetlinux-gpu-p3":1,' \
+ '"restricted-mxnetlinux-gpu-p3":1,"mxnetlinux-gpu-p3-8xlarge":1,"mxnetwindows-cpu":4,' \
+ '"mxnetwindows-gpu":1,"utility":30,"restricted-utility":30,' \
+ '"restricted-mxnetlinux-gpu":1}'
+ os.environ['JENKINS_PRIV_TUNNEL'] = 'localhost:48593'
+ os.environ['IGNORED_JENKINS_NODE_NAMES'] = '["master"]'
+ os.environ['WARM_POOL_SIZE'] = \
+ '{"mxnetlinux-cpu":1,"restricted-mxnetlinux-cpu":0,"mxnetlinux-gpu":0,"mxnetlinux-gpu-p3":0,' \
+ '"restricted-mxnetlinux-gpu-p3":0,"mxnetlinux-gpu-p3-8xlarge":0,"mxnetwindows-cpu":1,"mxnetwindows-gpu":0,' \
+ '"restricted-mxnetlinux-gpu":0}'
+ os.environ['MINIMUM_QUEUE_TIMES_SEC'] = \
+ '{"mxnetlinux-cpu":30,"restricted-mxnetlinux-cpu":30,"mxnetlinux-gpu":30,"mxnetlinux-gpu-p3":30,' \
+ '"restricted-mxnetlinux-gpu-p3":30,"mxnetlinux-gpu-p3-8xlarge":30,"mxnetwindows-cpu":30,' \
+ '"mxnetwindows-gpu":30,' \
+ '"restricted-mxnetlinux-gpu":30}'
+ os.environ['MANAGED_JENKINS_NODE_LABELS'] = \
+ '["mxnetlinux-cpu","restricted-mxnetlinux-cpu","mxnetlinux-gpu","mxnetwindows-cpu","mxnetwindows-gpu",' \
+ '"mxnetlinux-gpu-p3","restricted-mxnetlinux-gpu-p3","mxnetlinux-gpu-p3-8xlarge",' \
+ '"restricted-mxnetlinux-gpu"]'
+ os.environ['MAXIMUM_STARTUP_TIME_SEC'] = \
+ '{"mxnetlinux-cpu":300,"restricted-mxnetlinux-cpu":300,"mxnetlinux-gpu":300,"mxnetlinux-gpu-p3":300,' \
+ '"restricted-mxnetlinux-gpu-p3":300,"mxnetlinux-gpu-p3-8xlarge":300,"mxnetwindows-cpu":1800,' \
+ '"mxnetwindows-gpu":1800,'\
+ '"restricted-mxnetlinux-gpu": 300}'
+ os.environ['CCACHE_EFS_DNS'] = \
+ '{"mxnetlinux-cpu":"NONE","restricted-mxnetlinux-cpu":"NONE","mxnetlinux-gpu":"NONE",' \
+ '"mxnetlinux-gpu-p3":"NONE","restricted-mxnetlinux-gpu-p3":"NONE","mxnetlinux-gpu-p3-8xlarge":"NONE",' \
+ '"mxnetwindows-cpu":"NONE","mxnetwindows-gpu":"NONE", "restricted-mxnetlinux-gpu": "NONE"}'
+ os.environ['IGNORED_JENKINS_NODE_LABELS'] = '["mxnetlinux","mxnetwindows","master"]'
+
+ def test_get_starting_node_names(self):
+ connected_nodes = [
+ self._util_create_node(display_name='mxnetlinux-gpu10', is_offline=False, assigned_labels=[],
+ nb_executors=1, temporarily_offline=False),
+ self._util_create_node(display_name='mxnetlinux-cpu10', is_offline=False, assigned_labels=[],
+ nb_executors=1, temporarily_offline=False),
+ self._util_create_node(display_name='mxnetlinux-cpu11', is_offline=True, assigned_labels=[],
+ nb_executors=1, temporarily_offline=False),
+ self._util_create_node(display_name='mxnetlinux-gpu11', is_offline=True, assigned_labels=[],
+ nb_executors=1, temporarily_offline=False),
+ self._util_create_node(display_name='mxnetlinux-gpu12', is_offline=True, assigned_labels=[],
+ nb_executors=1, temporarily_offline=False),
+ self._util_create_node(display_name='mxnetlinux-gpu-temp-offline1', is_offline=True, assigned_labels=[],
+ nb_executors=1, temporarily_offline=True),
+ self._util_create_node(display_name='mxnetlinux-gpu-temp-offline2', is_offline=True, assigned_labels=[],
+ nb_executors=1, temporarily_offline=True),
+ ]
+
+ instance_running_durations = {
+ 'mxnetlinux-gpu10': 10, 'mxnetlinux-cpu10': 10, 'mxnetlinux-cpu11': 10, 'mxnetlinux-gpu11': 10,
+ 'mxnetlinux-gpu12': 10, 'mxnetlinux-gpu-temp-offline1': 10, 'mxnetlinux-gpu-temp-offline2': 10
+ }
+
+ # Data in placebo/test_metrics/test_get_starting_nodes
+ session = self._util_get_placebo_session(current_file=__file__, test_name='test_get_starting_nodes',
+ record=False)
+ ec2_resource = session.resource('ec2')
+
+ starting_nodes = autoscaling._unconnected_instances(nodes=connected_nodes, ec2_resource=ec2_resource,
+ instance_uptime=instance_running_durations)
+ assert len(starting_nodes['mxnetlinux-cpu']) == 1, \
+ 'Should be 1 starting cpu node, got {}'.format(starting_nodes['mxnetlinux-cpu'])
+ assert len(starting_nodes['mxnetlinux-gpu']) == 2, \
+ 'Should be 2 starting gpu nodes, got {}'.format(starting_nodes['mxnetlinux-gpu'])
+
+ def test_determine_scale_up_nodes(self):
+ def execute(nb_queue_items_cpu, nb_queue_items_gpu, nb_starting_nodes_cpu, nb_starting_nodes_gpu):
+ count_expected_error_messages = 0
+ nb_executors_cpu = autoscaling._get_nb_executors_per_label()['mxnetlinux-cpu']
+ nb_executors_gpu = autoscaling._get_nb_executors_per_label()['mxnetlinux-gpu']
+
+ queue_items = [
+ # Add some garbage that should not be recognized
+ self._util_create_queue_item(reason='Something mxnetlinux-gpu', in_queue_duration_sec=60000),
+ self._util_create_queue_item(reason='Something mxnetlinux-bla', in_queue_duration_sec=60000),
+
+ # Restricted jobs. It might be possible that there are jobs in the queue although there are nodes
+ # available.
+ # This might be happen if an unrestricted job tries to schedule a job onto a restricted node. This
+ # should not
+ # trigger a scale up but instead throw an error so somebody can investigate
+ self._util_create_queue_item(
+ reason=random.choice(QUEUE_WHY_FORMULATIONS).format('restricted-mxnetlinux-cpu'),
+ in_queue_duration_sec=60000),
+ ]
+ count_expected_error_messages += 1 # Restricted jobs
+
+ # Add some items that are not old enough and should thus not be picked up
+ NB_IMMATURE_QUEUE_ITEMS = 20
+ IMMATURE_QUEUE_DURATION_SEC = -15
+ for _ in range(0, NB_IMMATURE_QUEUE_ITEMS):
+ queue_items.append(self._util_create_queue_item(
+ reason=random.choice(QUEUE_WHY_FORMULATIONS).format('mxnetlinux-cpu'),
+ in_queue_duration_sec=IMMATURE_QUEUE_DURATION_SEC))
+
+ for _ in range(0, nb_queue_items_cpu):
+ queue_items.append(self._util_create_queue_item(
+ reason=random.choice(QUEUE_WHY_FORMULATIONS).format('mxnetlinux-cpu'),
+ in_queue_duration_sec=autoscaling._minimum_queue_times_sec()['mxnetlinux-cpu'] + 10))
+
+ for _ in range(0, nb_queue_items_gpu):
+ queue_items.append(self._util_create_queue_item(
+ reason=random.choice(QUEUE_WHY_FORMULATIONS).format('mxnetlinux-gpu'),
+ in_queue_duration_sec=autoscaling._minimum_queue_times_sec()['mxnetlinux-gpu'] + 10))
+
+ nodes = [
+ # Mis-configured node with multiple labels. Should be ignored.
+ self._util_create_node(display_name='multiple-assigned-labels-node',
+ assigned_labels=['mxnetlinux-cpu', 'mxnetlinux-gpu'],
+ nb_executors=10),
+
+ # Valid nodes
+ self._util_create_node(display_name='mxnet-linux-cpu10', assigned_labels=['mxnetlinux-cpu'],
+ nb_executors=nb_executors_cpu),
+ self._util_create_node(display_name='mxnet-linux-gpu10', assigned_labels=['garbage', 'mxnetlinux-gpu'],
+ nb_executors=nb_executors_gpu)
+ ]
+
+ starting_node_names = {
+ 'mxnetlinux-cpu': ['starting-cpu' + str(i) for i in range(nb_starting_nodes_cpu)],
+ 'mxnetlinux-gpu': ['starting-gpu' + str(i) for i in range(nb_starting_nodes_gpu)]
+ }
+
+ # Restricted nodes (see above for comment)
+ for i in range(5):
+ node = self._util_create_node(display_name='restricted-slave-' + str(i),
+ assigned_labels=
+ ['restricted-mxnetlinux-cpu'],
+ nb_executors=nb_executors_cpu, is_offline=False, is_idle=True)
+ nodes.append(node)
+
+ # Calculate estimated results
+ estimated_cpu = max(0, math.ceil(nb_queue_items_cpu / nb_executors_cpu) - nb_starting_nodes_cpu)
+ estimated_gpu = max(0, math.ceil(nb_queue_items_gpu / nb_executors_gpu) - nb_starting_nodes_gpu)
+
+ with self.assertLogs(level='ERROR') as log_manager:
+ scale_up_nb_nodes = autoscaling.determine_scale_up_nodes(
+ queue_items=queue_items,
+ nodes=nodes,
+ unconnected=starting_node_names)
+
+ assert len(log_manager.records) == count_expected_error_messages, \
+ "Expected {} log messages, got {}: {}". \
+ format(count_expected_error_messages, len(log_manager.records),
+ [record.message for record in log_manager.records])
+ logging.error("Hack to make sure the with-assert does not fail - we assert manually...")
+
+ if estimated_cpu > 0:
+ assert scale_up_nb_nodes['mxnetlinux-cpu'] == estimated_cpu, \
+ "Expected {} mxnetlinux-cpu nodes, got {}".format(estimated_cpu,
+ scale_up_nb_nodes['mxnetlinux-cpu'])
+ else:
+ assert 'mxnetlinux-cpu' not in scale_up_nb_nodes, \
+ "mxnetlinux-cpu should not be in scale_up_nb_nodes, got {}".format(
+ scale_up_nb_nodes['mxnetlinux-cpu'])
+
+ if estimated_gpu > 0:
+ assert scale_up_nb_nodes['mxnetlinux-gpu'] == estimated_gpu, \
+ "Expected {} mxnetlinux-gpu nodes, got {}".format(estimated_gpu,
+ scale_up_nb_nodes['mxnetlinux-gpu'])
+ else:
+ assert 'mxnetlinux-gpu' not in scale_up_nb_nodes, \
+ "mxnetlinux-gpu should not be in scale_up_nb_nodes, got {}".format(
+ scale_up_nb_nodes['mxnetlinux-gpu'])
+
+ execute(nb_queue_items_cpu=20, nb_queue_items_gpu=20, nb_starting_nodes_cpu=2, nb_starting_nodes_gpu=2)
+ execute(nb_queue_items_cpu=20, nb_queue_items_gpu=20, nb_starting_nodes_cpu=0, nb_starting_nodes_gpu=1)
+ execute(nb_queue_items_cpu=5, nb_queue_items_gpu=5, nb_starting_nodes_cpu=5, nb_starting_nodes_gpu=5)
+ execute(nb_queue_items_cpu=10, nb_queue_items_gpu=2, nb_starting_nodes_cpu=2, nb_starting_nodes_gpu=2)
+ execute(nb_queue_items_cpu=0, nb_queue_items_gpu=0, nb_starting_nodes_cpu=2, nb_starting_nodes_gpu=2)
+ execute(nb_queue_items_cpu=0, nb_queue_items_gpu=0, nb_starting_nodes_cpu=0, nb_starting_nodes_gpu=0)
+
+ def test_determine_scale_down_nodes(self):
+ def execute(nb_offline, nb_idle_linux, nb_non_idle_linux, nb_idle_windows_ready, nb_idle_windows_not_ready):
+ label_linux = 'mxnetlinux-cpu'
+ label_windows = 'mxnetwindows-cpu'
+
+ nb_executors_linux = autoscaling._get_nb_executors_per_label()[label_linux]
+ nb_executors_windows = autoscaling._get_nb_executors_per_label()[label_windows]
+
+ instance_running_durations = {}
+ nb_expected_error_messages = 0
+
+ nodes = []
+ for i in range(0, nb_offline):
+ nodes.append(self._util_create_node(display_name='offline' + str(i), assigned_labels=[label_linux],
+ nb_executors=nb_executors_linux, is_offline=True, is_idle=True))
+
+ for i in range(0, nb_idle_linux):
+ node = self._util_create_node(display_name='idle_linux' + str(i), assigned_labels=[label_linux],
+ nb_executors=nb_executors_linux, is_offline=False, is_idle=True)
+ instance_running_durations[node['displayName']] = random.randint(0, 500000)
+ nodes.append(node)
+
+ # Windows slave within the timeframe which allows shutdown due to hourly billing
+ for i in range(0, nb_idle_windows_ready):
+ node = self._util_create_node(display_name='idle_windows_ready' + str(i),
+ assigned_labels=[label_windows],
+ nb_executors=nb_executors_windows, is_offline=False, is_idle=True,
+ architecture='Windows NT (unknown) (amd64)')
+ instance_running_durations[node['displayName']] = \
+ random.randint(autoscaling.WINDOWS_MIN_PARTIAL_RUNTIME_SECONDS + 1, 60 * 60 - 1) + \
+ random.randint(0, 10) * (60 * 60)
+ nodes.append(node)
+
+ # Windows slave outside the timeframe which allows shutdown due to hourly billing
+ for i in range(0, nb_idle_windows_not_ready):
+ node = self._util_create_node(display_name='idle_windows_not_ready' + str(i),
+ assigned_labels=[label_windows],
+ nb_executors=nb_executors_windows, is_offline=False, is_idle=True,
+ architecture='Windows NT (unknown) (amd64)')
+ instance_running_durations[node['displayName']] = \
+ 1 + random.randint(0, 10) * (60 * 60)
+ nodes.append(node)
+
+ for i in range(0, nb_non_idle_linux):
+ node = self._util_create_node(display_name='non-idle' + str(i), assigned_labels=[label_linux],
+ nb_executors=nb_executors_linux, is_offline=False, is_idle=False)
+ instance_running_durations[node['displayName']] = random.randint(0, 500000)
+ nodes.append(node)
+
+ # Error testing: Windows slave without a known running duration
+ for i in range(0, 5):
+ nb_expected_error_messages += 1
+ node = self._util_create_node(display_name='idle_windows_no_duration' + str(i),
+ assigned_labels=[label_windows],
+ nb_executors=nb_executors_windows, is_offline=False, is_idle=True,
+ architecture='Windows NT (unknown) (amd64)')
+ nodes.append(node)
+
+ # Error testing: Slave without label
+ for i in range(0, 5):
+ nb_expected_error_messages += 1
+ node = self._util_create_node(display_name='slave_without_label' + str(i),
+ assigned_labels=[],
+ nb_executors=1, is_offline=False, is_idle=True)
+ nodes.append(node)
+
+ # Error testing: Slave with unmanaged label
+ for i in range(0, 5):
+ # Don't check this message since it's only debug. Just make sure this node does not get processed
+ node = self._util_create_node(display_name='slave_unmanaged_label' + str(i),
+ assigned_labels=
+ [random.choice(autoscaling._ignored_jenkins_node_labels())],
+ nb_executors=1, is_offline=False, is_idle=True)
+ nodes.append(node)
+
+ # Since offline and non-idle nodes should be ignored, we expect only nb_idle - buffer to be
+ # turned off. Additionally, we check that only windows slaves within the WINDOWS_MIN_PARTIAL_RUNTIME_SECONDS
+ # timeframe are turned off. The buffer should still consider idle windows nodes that are currently not
+ # eligible to be turned off.
+ estimated_linux_turned_off = max(nb_idle_linux - autoscaling._warm_pool_node_counts()[label_linux], 0)
+ estimated_windows_turned_off = \
+ max(nb_idle_windows_ready -
+ max(0, autoscaling._warm_pool_node_counts()[label_windows] - nb_idle_windows_not_ready), 0)
+
+ with self.assertLogs(level='ERROR') as log_manager:
+ scale_down_nodes = autoscaling.determine_scale_down_nodes(nodes, instance_running_durations)
+
+ assert len(log_manager.records) == nb_expected_error_messages, "Expected {} log messages, got {}: {}". \
+ format(nb_expected_error_messages, len(log_manager.records),
+ [record.message for record in log_manager.records])
+ logging.error("Hack to make sure the with-assert does not fail - we assert manually...")
+
+ estimated_length = 0
+
+ if estimated_linux_turned_off > 0:
+ estimated_length += 1
+ assert len(scale_down_nodes[label_linux]) == estimated_linux_turned_off, \
+ "Expected {} {} nodes to be turned off, got {}". \
+ format(nb_executors_linux, label_linux, len(scale_down_nodes[label_linux]))
+
+ if estimated_windows_turned_off > 0:
+ estimated_length += 1
+ assert len(scale_down_nodes[label_windows]) == estimated_windows_turned_off, \
+ "Expected {} {} nodes to be turned off, got {}". \
+ format(estimated_windows_turned_off, label_windows, len(scale_down_nodes[label_windows]))
+
+ assert len(scale_down_nodes) == estimated_length, "Expected {} scale down node types, got {}". \
+ format(estimated_length, len(scale_down_nodes))
+
+ execute(nb_offline=5, nb_idle_linux=5, nb_idle_windows_ready=5, nb_idle_windows_not_ready=5,
+ nb_non_idle_linux=5)
+ execute(nb_offline=5, nb_idle_linux=5, nb_idle_windows_ready=5, nb_idle_windows_not_ready=15,
+ nb_non_idle_linux=5)
+ execute(nb_offline=0, nb_idle_linux=5, nb_idle_windows_ready=0, nb_idle_windows_not_ready=0,
+ nb_non_idle_linux=5)
+ execute(nb_offline=5, nb_idle_linux=0, nb_idle_windows_ready=5, nb_idle_windows_not_ready=5,
+ nb_non_idle_linux=5)
+ execute(nb_offline=5, nb_idle_linux=0, nb_idle_windows_ready=5, nb_idle_windows_not_ready=3,
+ nb_non_idle_linux=5)
+ execute(nb_offline=0, nb_idle_linux=0, nb_idle_windows_ready=0, nb_idle_windows_not_ready=0,
+ nb_non_idle_linux=5)
+ execute(nb_offline=0, nb_idle_linux=1, nb_idle_windows_ready=1, nb_idle_windows_not_ready=1,
+ nb_non_idle_linux=0)
+ execute(nb_offline=0, nb_idle_linux=0, nb_idle_windows_ready=0, nb_idle_windows_not_ready=0,
+ nb_non_idle_linux=0)
+
+ def test_instance_uptime(self):
+ session = self._util_get_placebo_session(current_file=__file__, test_name='test_instance_uptime',
+ record=False)
+ ec2_resource = session.resource('ec2')
+
+ startup_times = {
+ 'node1': datetime.datetime(year=2018, month=4, day=1, hour=5),
+ 'node2': datetime.datetime(year=2018, month=3, day=1, hour=5, minute=10),
+ 'node3': datetime.datetime(year=2018, month=3, day=5, hour=15, minute=5)
+ }
+
+ results = autoscaling._instance_uptime(ec2_resource=ec2_resource)
+
+ assert len(results) == len(startup_times), 'Expected {} results, got {}'. \
+ format(len(startup_times), len(results))
+
+ current_time = datetime.datetime.now()
+ for name, startup_time in startup_times.items():
+ expected_duration = (current_time - startup_time).total_seconds()
+ returned_duration = results[name]
+ duration_diff = abs(returned_duration - expected_duration)
+ assert duration_diff < 10, 'Duration {}s of {} does not match expected {}s. Diff: {}s'. \
+ format(returned_duration, name, expected_duration, duration_diff)
+
+ def test_calculate_nb_required_nodes(self):
+ def execute(nb_linux_cpu, nb_linux_gpu):
+ dict_required_executors = dict()
+
+ # Add some garbage
+ dict_required_executors['mxnetlinux-cpu111'] = 100
+ dict_required_executors['blabla'] = 100
+
+ # Should be ignored
+ dict_required_executors['mxnetlinux'] = 100
+
+ dict_required_executors['mxnetlinux-cpu'] = nb_linux_cpu
+ dict_required_executors['mxnetlinux-gpu'] = nb_linux_gpu
+
+ expected_linux_cpu = \
+ max(0, math.ceil(nb_linux_cpu / autoscaling._get_nb_executors_per_label()['mxnetlinux-cpu']))
+ expected_linux_gpu = \
+ max(0, math.ceil(nb_linux_gpu / autoscaling._get_nb_executors_per_label()['mxnetlinux-gpu']))
+
+ with self.assertLogs(level='ERROR') as log_manager:
+ dict_required_nodes = autoscaling._calculate_nb_required_nodes(
+ dict_required_executors=dict_required_executors)
+
+ # Expect 4 error messages for the two times of garbage
+ assert len(log_manager.records) == 4, "Expected 4 log messages, got {}: {}". \
+ format(len(log_manager.records), [record.message for record in log_manager.records])
+
+ if expected_linux_cpu > 0:
+ assert dict_required_nodes['mxnetlinux-cpu'] == expected_linux_cpu, \
+ "Expected {} mxnetlinux-cpu nodes, got {}". \
+ format(expected_linux_cpu, dict_required_nodes['mxnetlinux-cpu'])
+ else:
+ assert 'mxnetlinux-cpu' not in dict_required_nodes, "Expected no mxnetlinux-cpu nodes, got {}". \
+ format(dict_required_nodes['mxnetlinux-cpu'])
+
+ if expected_linux_gpu > 0:
+ assert dict_required_nodes['mxnetlinux-gpu'] == expected_linux_gpu, \
+ "Expected {} mxnetlinux-gpu nodes, got {}". \
+ format(expected_linux_gpu, dict_required_nodes['mxnetlinux-gpu'])
+ else:
+ assert 'mxnetlinux-gpu' not in dict_required_nodes, "Expected no mxnetlinux-gpu nodes, got {}". \
+ format(dict_required_nodes['mxnetlinux-gpu'])
+
+ execute(nb_linux_cpu=0, nb_linux_gpu=0)
+ execute(nb_linux_cpu=1, nb_linux_gpu=1)
+ execute(nb_linux_cpu=100, nb_linux_gpu=100)
+ execute(nb_linux_cpu=0, nb_linux_gpu=100)
+ execute(nb_linux_cpu=100, nb_linux_gpu=0)
+ execute(nb_linux_cpu=10, nb_linux_gpu=5)
+ execute(nb_linux_cpu=0, nb_linux_gpu=5)
+ execute(nb_linux_cpu=0, nb_linux_gpu=100)
+
+ def test_extract_type_label_from_queue_item(self):
+ # Invalid reason
+ def execute_invalid_reason():
+ # Invalid reasons should not log an error but just return None. There are a lot of valid other queue reasons
+ reason = 'This is a random queue reason with a {} label'. \
+ format(random.choice(list(autoscaling._managed_jenkins_node_labels())))
+ queue_item = self._util_create_queue_item(reason=reason, in_queue_duration_sec=0)
+ label = autoscaling._label_from_queued_job(nodes=[], queue_item=queue_item)
+ assert label is None, "Expected {} as label for '{}', got {}". \
+ format(None, reason, label)
+
+ # All reasons with valid labels
+ def execute_all_reasons_valid_labels():
+ for queue_formulation in QUEUE_WHY_FORMULATIONS:
+ expected_label = random.choice(list(autoscaling._managed_jenkins_node_labels()))
+ reason = queue_formulation.format(expected_label)
+ queue_item = self._util_create_queue_item(reason=reason, in_queue_duration_sec=0)
+ label = autoscaling._label_from_queued_job(nodes=[], queue_item=queue_item)
+ assert label == expected_label, "Expected {} as label for '{}', got {}". \
+ format(expected_label, reason, label)
+
+ # Reason with invalid label
+ def execute_one_reason_invalid_labels():
+ garbage_label = "something123"
+ reason = random.choice(QUEUE_WHY_FORMULATIONS).format(garbage_label)
+ queue_item = self._util_create_queue_item(reason=reason, in_queue_duration_sec=0)
+
+ with self.assertLogs(level='ERROR') as log_manager:
+ label = autoscaling._label_from_queued_job(nodes=[], queue_item=queue_item)
+
+ # Expect 1 error message, describing it could not associate label with a node and thus marking it as invalid
+ assert len(log_manager.records) == 1, "Expected 1 log message, got {}: {}". \
+ format(len(log_manager.records), [record.message for record in log_manager.records])
+
+ assert label is None, "Expected {} as label for '{}', got {}". \
+ format(None, reason, label)
+
+ # Reason which contains existing node name
+ def execute_reason_existing_node_name():
+ valid_label = random.choice(list(autoscaling._managed_jenkins_node_labels()))
+ node_name = valid_label + '10'
+ nodes = [self._util_create_node(display_name=node_name, assigned_labels=[valid_label], nb_executors=1)]
+ reason = random.choice(QUEUE_WHY_FORMULATIONS).format(node_name)
+ queue_item = self._util_create_queue_item(reason=reason, in_queue_duration_sec=0)
+
+ label = autoscaling._label_from_queued_job(nodes=nodes, queue_item=queue_item)
+
+ assert label == valid_label, "Expected {} as label for '{}', got {}". \
+ format(valid_label, reason, label)
+
+ # Reason which contains non-existing node name
+ def execute_reason_nonexisting_node_name():
+ # Similar to execute_one_reason_invalid_labels, just going with node data in this case.
+ node_name = 'mxnetwindows-cpu10'
+ reason = random.choice(QUEUE_WHY_FORMULATIONS).format(node_name)
+
+ # Fake nodes data, it should not matter and never be used since it does not match
+ nodes = [self._util_create_node(display_name="mxnetlinux-cpu10", assigned_labels=['mxnetlinux-cpu'],
+ nb_executors=1)]
+
+ queue_item = self._util_create_queue_item(reason=reason, in_queue_duration_sec=0)
+
+ with self.assertLogs(level='ERROR') as log_manager:
+ label = autoscaling._label_from_queued_job(nodes=nodes, queue_item=queue_item)
+
+ # Expect 1 error message, describing it could not associate label with a node and thus marking it as invalid
+ assert len(log_manager.records) == 1, "Expected 1 log message, got {}: {}". \
+ format(len(log_manager.records), [record.message for record in log_manager.records])
+
+ assert label is None, "Expected {} as label for '{}', got {}". \
+ format(None, reason, label)
+
+ # Reason which contains existing node name but node has no labels
+ def execute_reason_existing_node_no_labels():
+ node_name = 'mxnetlinux-cpu10'
+ nodes = [self._util_create_node(display_name=node_name, assigned_labels=[], nb_executors=1)]
+ reason = random.choice(QUEUE_WHY_FORMULATIONS).format(node_name)
+ queue_item = self._util_create_queue_item(reason=reason, in_queue_duration_sec=0)
+
+ with self.assertLogs(level='ERROR') as log_manager:
+ label = autoscaling._label_from_queued_job(nodes=nodes, queue_item=queue_item)
+
+ # Expect 1 error message, describing it could not extract valid label from that node
+ assert len(log_manager.records) == 1, "Expected 1 log message, got {}: {}". \
+ format(len(log_manager.records), [record.message for record in log_manager.records])
+
+ assert label is None, "Expected {} as label for '{}', got {}". \
+ format(None, reason, label)
+
+ # Special case when the jenkins master contains no nodes
+ def execute_no_running_nodes():
+ reason = 'Waiting for next available executor'
+ queue_item = self._util_create_queue_item(reason=reason, in_queue_duration_sec=0)
+ with self.assertLogs(level='DEBUG') as log_manager:
+ label = autoscaling._label_from_queued_job(nodes=[], queue_item=queue_item)
+
+ # Expect 1 error message, describing it could not extract valid label from that node
+ assert len(log_manager.records) == 1, "Expected 1 log message, got {}: {}". \
+ format(len(log_manager.records), [record.message for record in log_manager.records])
+
+ assert label == 'mxnetlinux-cpu', "Expected {} as label for '{}', got {}". \
+ format('mxnetlinux-cpu', reason, label)
+
+ execute_invalid_reason()
+ execute_all_reasons_valid_labels()
+ execute_one_reason_invalid_labels()
+ execute_reason_existing_node_name()
+ execute_reason_nonexisting_node_name()
+ execute_reason_existing_node_no_labels()
+ execute_no_running_nodes()
+
+ # Create mock class for test_mark_nodes_offline/online. Has to be at this scope due to pickle restrictions
+ class MockMarkNodesClass(object):
+ def __init__(self):
+ self.called = False
+ self.reason = None
+ self.name = None
+
+ def set_offline(self, reason):
+ self.called = True
+ self.reason = reason
+
+ def set_online(self):
+ self.called = True
+
+ def test_mark_nodes_offline(self):
+ def execute(nb_nodes):
+ nodes = []
+ for _ in range(0, nb_nodes):
+ nodes.append(self.MockMarkNodesClass())
+
+ offline_reason = 'RANDOM REASON'
+ autoscaling._mark_nodes_offline(offline_nodes=nodes, reason=offline_reason)
+
+ for node_obj in nodes:
+ assert node_obj.called, "Expected set_offline to be called"
+ assert node_obj.reason == offline_reason, "Expected {} as offline reason, got {}". \
+ format(offline_reason, node_obj.reason)
+
+ execute(0)
+ execute(1)
+ execute(10)
+ execute(20)
+ execute(50)
+ execute(150)
+ execute(500)
+ execute(5000)
+
+ def test_mark_nodes_online(self):
+ def execute(nb_nodes):
+ nodes = []
+ for _ in range(0, nb_nodes):
+ nodes.append(self.MockMarkNodesClass())
+
+ autoscaling._mark_nodes_online(online_nodes=nodes)
+
+ for node_obj in nodes:
+ assert node_obj.called, "Expected set_online to be called"
+
+ execute(0)
+ execute(1)
+ execute(10)
+ execute(20)
+ execute(50)
+ execute(150)
+ execute(500)
+ execute(5000)
+
+ # Create mock class for test_mark_nodes_offline. Has to be at this scope due to pickle restrictions
+ class MockConvertJenkinsNodeServer(object):
+ def __init__(self):
+ self.baseurl = 'http://0.0.0.0/'
+
+ def test_convert_jenkins_nodes(self):
+ def execute(nb_nodes):
+ MAGIC_RETURN_VALUE = {'magic': True}
+ mock_server = self.MockConvertJenkinsNodeServer()
+
+ node_names = set()
+ for _ in range(0, nb_nodes):
+ node_names.add(''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(20)))
+
+ expected_nb_nodes_obj = len(node_names)
+
+ # Test whether poll was called as often as we have nodes. This ensures the parallelization is working
+ # and no additional requests are being sent
+ with unittest.mock.patch.object(jenkinsapi.jenkinsbase.JenkinsBase, attribute='_poll') as mock_method:
+ call_count = 0
+
+ # pylint: disable=unused-argument
+ def increment_call_count(**kwargs):
+ nonlocal call_count
+ with MOCK_CONVERT_JENKINS_NODES_THREAD_LOCK:
+ call_count += 1
+
+ return MAGIC_RETURN_VALUE
+
+ # We can't use mock_method.call_count because it is not threadsafe
+ mock_method.side_effect = increment_call_count
+ nodes_obj = autoscaling._convert_to_jenkins_nodes(server=mock_server, instances=list(node_names))
+
+ for name, node_obj in nodes_obj.items():
+ assert node_obj._data == MAGIC_RETURN_VALUE, 'Poll has not been called for {}'.format(name)
+ node_names.remove(name) # Throws an exception if name does not exist. This should not happen
+
+ assert call_count == nb_nodes, 'Expected {} poll calls, got {}'. \
+ format(nb_nodes, mock_method.call_count)
+ assert len(nodes_obj) == expected_nb_nodes_obj, 'Expected {} resulting objects, got {}'. \
+ format(expected_nb_nodes_obj, len(nodes_obj))
+
+ execute(nb_nodes=0)
+ execute(nb_nodes=1)
+ execute(nb_nodes=5)
+ execute(nb_nodes=10)
+ execute(nb_nodes=50)
+ execute(nb_nodes=150)
+ execute(nb_nodes=500)
+ execute(nb_nodes=5000)
+
+ class MockRemoveNonIdleOfflineNodes(object):
+ def __init__(self, is_idle, is_online):
+ self.nb_updates = 0
+ self.idle = is_idle
+ self.online = is_online
+
+ def poll(self):
+ with MOCK_FILTER_NON_IDLE_OFFLINE_NODES_THREAD_LOCK:
+ self.nb_updates += 1
+
+ def is_idle(self):
+ return self.idle
+
+ def is_online(self):
+ return self.online
+
+ def test_filter_non_idle_offline_nodes(self):
+ def execute(nb_idle_offline, nb_non_idle_offline, nb_idle_online, nb_non_idle_online):
+ node_objs = []
+
+ for _ in range(0, nb_idle_offline):
+ node_objs.append(self.MockRemoveNonIdleOfflineNodes(is_idle=True, is_online=False))
+ for _ in range(0, nb_non_idle_offline):
+ node_objs.append(self.MockRemoveNonIdleOfflineNodes(is_idle=False, is_online=False))
+ for _ in range(0, nb_idle_online):
+ node_objs.append(self.MockRemoveNonIdleOfflineNodes(is_idle=True, is_online=True))
+ for _ in range(0, nb_non_idle_online):
+ node_objs.append(self.MockRemoveNonIdleOfflineNodes(is_idle=False, is_online=True))
+
+ with self.assertLogs(level='INFO') as log_manager:
+ filtered_node_objs, non_idle_nodes = autoscaling._partition_non_idle(node_objs)
+
+ # Expect 1 warning message if there were any non-idle or online nodes
+ expected_messages = 1 if (nb_non_idle_offline + nb_idle_online + nb_non_idle_online) > 0 else 0
+ assert len(log_manager.records) == expected_messages, "Expected {} log message, got {}: {}". \
+ format(expected_messages, len(log_manager.records),
+ [record.message for record in log_manager.records])
+ logging.warning("Hack to make sure the with-assert does not fail - we assert manually...")
+
+ assert len(filtered_node_objs) == nb_idle_offline, "Expected {} filtered nodes, got {}". \
+ format(nb_idle_offline, len(filtered_node_objs))
+
+ nb_filtered = nb_non_idle_offline + nb_idle_online + nb_non_idle_online
+ assert len(non_idle_nodes) == nb_filtered, "Expected {} unfiltered nodes, got {}". \
+ format(nb_non_idle_offline, len(non_idle_nodes))
+
+ for node_obj in filtered_node_objs:
+ assert node_obj.nb_updates == 1, "Expected 1 node update, got {}".format(node_obj.nb_updates)
+
+ execute(nb_idle_offline=5, nb_non_idle_offline=5, nb_idle_online=5, nb_non_idle_online=5)
+ execute(nb_idle_offline=0, nb_non_idle_offline=5, nb_idle_online=5, nb_non_idle_online=5)
+ execute(nb_idle_offline=50, nb_non_idle_offline=0, nb_idle_online=0, nb_non_idle_online=0)
+ execute(nb_idle_offline=5, nb_non_idle_offline=0, nb_idle_online=0, nb_non_idle_online=5)
+ execute(nb_idle_offline=5000, nb_non_idle_offline=5000, nb_idle_online=5000, nb_non_idle_online=5000)
+ execute(nb_idle_offline=0, nb_non_idle_offline=0, nb_idle_online=0, nb_non_idle_online=0)
+
+ def test_launch_ec2_instances(self):
+ special_case_in_exception = '12345678901234567890throwexception'
+
+ class MockEc2RunInstancesClass(object):
+ def __init__(self):
+ # Small hack to get to meta.client.run_instances
+ self.meta = self
+ self.meta.client = self
+ self.calls = []
+
+ def run_instances(self, **kwargs):
+ for tag_dict in kwargs['TagSpecifications'][0]['Tags']:
+ if tag_dict['Key'] == 'Name' and tag_dict['Value'] == special_case_in_exception:
+ # Special case which should throw an exception
+ raise ClientError(error_response=unittest.mock.MagicMock(), operation_name='test2')
+
+ self.calls.append(kwargs)
+
+ def execute(nb_invalid_labels, nb_valid_entries):
+ valid_label = random.choice(list(autoscaling._managed_jenkins_node_labels()))
+ scale_up_slots = defaultdict(list)
+ expected_messages = nb_invalid_labels
+
+ # Invalid labels
+ for _ in range(0, nb_invalid_labels):
+ invalid_label = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(20))
+ instance_name = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(20))
+ scale_up_slots[invalid_label].append(instance_name)
+
+ # Valid entries
+ for _ in range(0, nb_valid_entries):
+ instance_name = valid_label + '_' + \
+ ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(20))
+ scale_up_slots[valid_label].append(instance_name)
+
+ # Make a special case which will result in an exception, thrown by our mock.
+ scale_up_slots[valid_label].append(special_case_in_exception)
+ expected_messages += 1
+
+ run_instances_mock = MockEc2RunInstancesClass()
+ with self.assertLogs(level='ERROR') as log_manager:
+ started_instances = autoscaling._launch_ec2_instances(scale_up_slots, run_instances_mock)
+ assert len(log_manager.records) == expected_messages, "Expected {} log message, got {}: {}". \
+ format(expected_messages, len(log_manager.records),
+ [record.message for record in log_manager.records])
+ logging.error("Hack to make sure the with-assert does not fail - we assert manually...")
+
+ assert len(run_instances_mock.calls) == nb_valid_entries, 'Expected {} run_instances_mock.calls, got {}'. \
+ format(nb_valid_entries, len(run_instances_mock.calls))
+ assert len(started_instances) == nb_valid_entries, 'Expected {} started instances, got {}'. \
+ format(nb_valid_entries, len(started_instances))
+ # TODO: Validate content of these calls (e.g. label being set, right template etc)
+
+ execute(nb_invalid_labels=5, nb_valid_entries=5)
+ execute(nb_invalid_labels=0, nb_valid_entries=50)
+ execute(nb_invalid_labels=50, nb_valid_entries=0)
+ execute(nb_invalid_labels=0, nb_valid_entries=0)
+ execute(nb_invalid_labels=50, nb_valid_entries=50)
+
+ def test_format_ec2_user_data_command(self):
+ with self.assertLogs(level='ERROR') as log_manager:
+ for managed_label in autoscaling._managed_jenkins_node_labels():
+ command = autoscaling._format_ec2_user_data_command(label=managed_label,
+ target_instance_name='instance')
+ assert command is not None, "Expected a user data command for {}, got None".format(managed_label)
+
+ assert not log_manager.records, "Expected {} log message, got {}: {}". \
+ format(0, len(log_manager.records), [record.message for record in log_manager.records])
+
+ # Now input a non-existent label and ensure an error is getting printed
+ command = autoscaling._format_ec2_user_data_command(label='garbage-label', target_instance_name='instance')
+ assert command is None, "Expected None as user data command, got {}".format(command)
+ assert len(log_manager.records) == 1, "Expected {} log message, got {}: {}". \
+ format(1, len(log_manager.records), [record.message for record in log_manager.records])
+
+ def test_create_jenkins_node_slots(self):
+ def execute(nb_invalid_label, nb_valid_instances):
+ jenkins_server = MockJenkinsServer(baseurl='http://0.0.0.0/')
+ scale_up_nb_nodes = dict()
+ valid_label = random.choice(list(autoscaling._get_slave_configuration()))
+
+ # Invalid label without config
+ for _ in range(0, nb_invalid_label):
+ invalid_label = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(20))
+ scale_up_nb_nodes[invalid_label] = random.randint(1, 10)
+
+ # Valid label
+ scale_up_nb_nodes[valid_label] = nb_valid_instances
+
+ with self.assertLogs(level='ERROR') as log_manager:
+ node_slots = autoscaling._create_jenkins_node_slots(jenkins_server, scale_up_nb_nodes)
+ expected_messages = nb_invalid_label
+ assert len(log_manager.records) == expected_messages, "Expected {} log message, got {}: {}". \
+ format(expected_messages, len(log_manager.records),
+ [record.message for record in log_manager.records])
+ logging.error("Hack to make sure the with-assert does not fail - we assert manually...")
+
+ if nb_valid_instances > 0:
+ assert len(node_slots[valid_label]) == nb_valid_instances, \
+ "Expected {} valid instances, got {}".format(nb_valid_instances, len(node_slots[valid_label]))
+ else:
+ assert valid_label not in node_slots, \
+ "Expected 0 valid instances, got {}".format(len(node_slots[valid_label]))
+
+ # Check if the invalid slots have been processed
+ expected_node_slots_length = 1 if nb_valid_instances > 0 else 0
+ assert len(node_slots) == expected_node_slots_length, \
+ "Expected {} valid node slot label, got {}".format(expected_node_slots_length, len(node_slots))
+
+ # TODO: Validate post requests
+ assert jenkins_server.nb_post_called == nb_valid_instances, "Expected {} slot creations, got {}". \
+ format(nb_valid_instances, jenkins_server.nb_post_called)
+
+ execute(nb_invalid_label=5, nb_valid_instances=5)
+ execute(nb_invalid_label=5000, nb_valid_instances=5000)
+ execute(nb_invalid_label=0, nb_valid_instances=0)
+ execute(nb_invalid_label=5, nb_valid_instances=0)
+ execute(nb_invalid_label=0, nb_valid_instances=5)
+
+ def test_extract_node_type_label(self):
+ def execute(log_threshhold, nb_log_messages, assigned_labels, expected_label):
+ data = self._util_create_node(display_name='invalid', nb_executors=1, assigned_labels=assigned_labels)
+
+ with self.assertLogs(level=log_threshhold) as log_manager:
+ type_label = autoscaling._managed_node_label(data)
+ expected_messages = nb_log_messages
+ assert len(log_manager.records) == expected_messages, "Expected {} log message, got {}: {}". \
+ format(expected_messages, len(log_manager.records),
+ [record.message for record in log_manager.records])
+ logging.error("Hack to make sure the with-assert does not fail - we assert manually...")
+
+ assert type_label == expected_label, \
+ 'Expected {} as returned label, got {}'.format(expected_label, type_label)
+
+ def launch_blacklisted_label():
+ blacklisted_label = random.choice(list(autoscaling._ignored_jenkins_node_labels()))
+ execute(log_threshhold='DEBUG', nb_log_messages=1, assigned_labels=[blacklisted_label],
+ expected_label=blacklisted_label)
+
+ valid_label = random.choice(list(autoscaling._managed_jenkins_node_labels()))
+ execute(log_threshhold='DEBUG', nb_log_messages=1, assigned_labels=[valid_label, blacklisted_label],
+ expected_label=blacklisted_label)
+ execute(log_threshhold='DEBUG', nb_log_messages=1, assigned_labels=[blacklisted_label, valid_label],
+ expected_label=blacklisted_label)
+
+ random_label = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(20))
+ execute(log_threshhold='DEBUG', nb_log_messages=1,
+ assigned_labels=[valid_label, blacklisted_label, random_label],
+ expected_label=blacklisted_label)
+ execute(log_threshhold='DEBUG', nb_log_messages=1,
+ assigned_labels=[blacklisted_label, random_label, valid_label],
+ expected_label=blacklisted_label)
+ execute(log_threshhold='DEBUG', nb_log_messages=1,
+ assigned_labels=[random_label, blacklisted_label, valid_label],
+ expected_label=blacklisted_label)
+
+ def launch_multiple_managed_label():
+ valid_labels_iter = iter(autoscaling._managed_jenkins_node_labels())
+ valid_label_1 = valid_labels_iter.__next__()
+ valid_label_2 = valid_labels_iter.__next__()
+ random_label = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(20))
+
+ execute(log_threshhold='ERROR', nb_log_messages=1, assigned_labels=[valid_label_1, valid_label_2],
+ expected_label=None)
+ execute(log_threshhold='ERROR', nb_log_messages=1, assigned_labels=[valid_label_2, valid_label_1],
+ expected_label=None)
+
+ execute(log_threshhold='ERROR', nb_log_messages=1,
+ assigned_labels=[random_label, valid_label_1, valid_label_2],
+ expected_label=None)
+ execute(log_threshhold='ERROR', nb_log_messages=1,
+ assigned_labels=[valid_label_2, random_label, valid_label_1],
+ expected_label=None)
+
+ def launch_no_managed_labels():
+ execute(log_threshhold='WARNING', nb_log_messages=1, assigned_labels=[
+ ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(20)),
+ ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(20)),
+ ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(20)),
+ ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(20))],
+ expected_label=None)
+
+ def launch_no_labels():
+ execute(log_threshhold='WARNING', nb_log_messages=1, assigned_labels=[], expected_label=None)
+
+ def launch_one_managed_label():
+ label = random.choice(list(autoscaling._managed_jenkins_node_labels()))
+ random_label = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(20))
+ execute(log_threshhold='WARNING', nb_log_messages=0, assigned_labels=[label], expected_label=label)
+ execute(log_threshhold='WARNING', nb_log_messages=0, assigned_labels=[label, random_label],
+ expected_label=label)
+ execute(log_threshhold='WARNING', nb_log_messages=0, assigned_labels=[random_label, label],
+ expected_label=label)
+
+ launch_blacklisted_label()
+ launch_multiple_managed_label()
+ launch_no_managed_labels()
+ launch_no_labels()
+ launch_one_managed_label()
+
+ def test_delete_jenkins_node_objects(self):
+ class MockJenkinsNode(object):
+ def __init__(self, jenkins, name, is_online):
+ self.online = is_online
+ self.name = name
+ self.baseurl = 'http://0.0.0.0/'
+ self.jenkins = jenkins
+
+ def is_online(self):
+ return self.online
+
+ def execute(nb_online, nb_offline):
+ jenkins_server = MockJenkinsServer(baseurl='http://0.0.0.0/')
+ node_objs = {}
+
+ for _ in range(0, nb_online):
+ name = 'online' + ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(20))
+ node_objs[name] = MockJenkinsNode(name=name, jenkins=jenkins_server, is_online=True)
+
+ for _ in range(0, nb_offline):
+ name = 'offline' + ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(20))
+ node_objs[name] = MockJenkinsNode(name=name, jenkins=jenkins_server, is_online=False)
+
+ with self.assertLogs(level='ERROR') as log_manager:
+ autoscaling._delete_jenkins_node_objects(node_objs)
+ expected_messages = nb_online
+ assert len(log_manager.records) == expected_messages, "Expected {} log message, got {}: {}". \
+ format(expected_messages, len(log_manager.records),
+ [record.message for record in log_manager.records])
+ logging.error("Hack to make sure the with-assert does not fail - we assert manually...")
+
+ assert jenkins_server.nb_post_called == nb_offline, "Expected {} slave deletions, got {}". \
+ format(nb_offline, jenkins_server.nb_post_called)
+
+ execute(nb_offline=0, nb_online=0)
+ execute(nb_offline=5, nb_online=5)
+ execute(nb_offline=0, nb_online=10)
+ execute(nb_offline=10, nb_online=0)
+ execute(nb_offline=1000, nb_online=0)
+ execute(nb_offline=0, nb_online=1000)
+ execute(nb_offline=1000, nb_online=1000)
+
+ # pylint: disable=no-self-use
+ def test_apply_upscale_limit(self):
+ def execute(limit, nb_scale_up_nodes, nb_labels):
+ nodes = dict()
+ nodes['empty'] = 0
+ slots_left = nb_scale_up_nodes
+ for i in range(0, nb_labels):
+ label = 'label' + str(i)
+ if i == nb_labels - 1:
+ nb_nodes = slots_left
+ else:
+ nb_nodes = random.randint(0, slots_left)
+ slots_left -= nb_nodes
+ nodes[label] = nb_nodes
+
+ new_scale_up_nb_nodes = autoscaling._apply_upscale_limit(limit, nodes)
+
+ nb_total_nodes = sum(new_scale_up_nb_nodes.values())
+ assert nb_total_nodes <= limit, "Expected {} nodes at most, got {}. Nodes: {}". \
+ format(limit, nb_total_nodes, pprint.pprint(new_scale_up_nb_nodes))
+ if nb_scale_up_nodes >= limit:
+ assert nb_total_nodes == limit, "Expected {} nodes, got {}. Nodes: {}". \
+ format(limit, nb_total_nodes, pprint.pprint(new_scale_up_nb_nodes))
+ else:
+ assert nb_total_nodes == nb_scale_up_nodes, "Expected {} nodes, got {}. Nodes: {}". \
+ format(nb_scale_up_nodes, nb_total_nodes, pprint.pprint(new_scale_up_nb_nodes))
+
+ # Ensure we got no unexpected labels back
+ for label in new_scale_up_nb_nodes.keys():
+ assert label in nodes.keys(), "Found unexpected label {}. Expected labels: {}". \
+ format(label, nodes.keys())
+
+ execute(limit=10, nb_scale_up_nodes=10, nb_labels=5)
+ execute(limit=1, nb_scale_up_nodes=10, nb_labels=5)
+ execute(limit=10, nb_scale_up_nodes=10, nb_labels=1)
+ execute(limit=10, nb_scale_up_nodes=50, nb_labels=1)
+ for i in range(0, 100):
+ execute(limit=10, nb_scale_up_nodes=i, nb_labels=5)
+
+ # pylint: disable=no-self-use
+ def test_apply_downscale_limit(self):
+ def execute(limit, nb_scale_down_nodes, nb_labels):
+ nodes = dict()
+ nodes['empty'] = list()
+ slots_left = nb_scale_down_nodes
+ for i in range(0, nb_labels):
+ label = 'label' + str(i)
+ if i == nb_labels - 1:
+ nb_nodes = slots_left
+ else:
+ nb_nodes = random.randint(0, slots_left)
+ slots_left -= nb_nodes
+
+ nodes[label] = list()
+ for j in range(0, nb_nodes):
+ nodes[label].append(j) # j is a dummy object
+
+ new_scale_down_nodes = autoscaling._apply_downscale_limit(limit, scale_down_nodes=nodes)
+
+ nb_total_nodes = sum([len(lst) for lst in new_scale_down_nodes.values()])
+ assert nb_total_nodes <= limit, "Expected {} nodes at most, got {}. Nodes: {}". \
+ format(limit, nb_total_nodes, pprint.pformat(new_scale_down_nodes))
+ if nb_total_nodes == limit:
+ assert nb_total_nodes == limit, "Expected {} nodes, got {}. Nodes: {}". \
+ format(limit, nb_total_nodes, pprint.pformat(new_scale_down_nodes))
+
+ execute(limit=10, nb_scale_down_nodes=10, nb_labels=5)
+ execute(limit=1, nb_scale_down_nodes=10, nb_labels=5)
+ execute(limit=10, nb_scale_down_nodes=10, nb_labels=1)
+ execute(limit=10, nb_scale_down_nodes=50, nb_labels=5)
+ execute(limit=10, nb_scale_down_nodes=50, nb_labels=1)
+
+ def test_determine_faulty_nodes(self):
+ nodes_data = [
+ # Master
+ self._util_create_node(display_name='master', assigned_labels=['master'],
+ nb_executors=0, temporarily_offline=False, is_offline=False),
+
+ # OK instances
+ self._util_create_node(display_name='fresh_instance1', assigned_labels=['mxnetlinux-cpu'],
+ nb_executors=1, temporarily_offline=False, is_offline=False),
+ self._util_create_node(display_name='fresh_instance2', assigned_labels=['mxnetlinux-cpu'],
+ nb_executors=1, temporarily_offline=False, is_offline=False),
+ self._util_create_node(display_name='fresh_instance3', assigned_labels=['mxnetlinux-cpu'],
+ nb_executors=1, temporarily_offline=False, is_offline=True,
+ offline_cause_class='hudson.node_monitors.ResponseTimeMonitor$Data',
+ offline_reason='Timed out for last 5 attempts'),
+
+ # Dont shut down manually taken offline instances
+ self._util_create_node(display_name='offline_ignore1', assigned_labels=['mxnetlinux-cpu'],
+ nb_executors=1, temporarily_offline=True, is_offline=False,
+ offline_reason='Custom offline1'),
+ self._util_create_node(display_name='offline_ignore2', assigned_labels=['mxnetlinux-cpu'],
+ nb_executors=1, temporarily_offline=True, is_offline=False,
+ offline_reason='Custom offline2'),
+
+ # Timeout during start
+ self._util_create_node(display_name='start_timeout1', assigned_labels=['mxnetlinux-cpu'],
+ nb_executors=1, temporarily_offline=False, is_offline=False),
+ self._util_create_node(display_name='start_timeout2', assigned_labels=['mxnetlinux-cpu'],
+ nb_executors=1, temporarily_offline=False, is_offline=False),
+ self._util_create_node(display_name='start_timeout3', assigned_labels=['mxnetlinux-cpu'],
+ nb_executors=1, temporarily_offline=False, is_offline=False),
+
+ # Taken offline (e.g. low disk space)
+ self._util_create_node(display_name='taken_offline_unhealthy1', assigned_labels=['mxnetlinux-cpu'],
+ nb_executors=1, temporarily_offline=True, is_offline=False,
+ offline_cause_class='hudson.node_monitors.DiskSpaceMonitorDescriptor$DiskSpace',
+ offline_reason='Disk space is too low. Only 49.633GB left on /home/jenkins_slave.'),
+ self._util_create_node(display_name='taken_offline_unhealthy2', assigned_labels=['mxnetlinux-cpu'],
+ nb_executors=1, temporarily_offline=True, is_offline=False,
+ offline_cause_class='hudson.node_monitors.DiskSpaceMonitorDescriptor$DiskSpace',
+ offline_reason='Disk space is too low. Only 49.633GB left on /home/jenkins_slave.'),
+ self._util_create_node(display_name='taken_offline_unhealthy3', assigned_labels=['mxnetlinux-cpu'],
+ nb_executors=1, temporarily_offline=True, is_offline=False,
+ offline_cause_class='hudson.node_monitors.Something$Something',
+ offline_reason='Something is unhealthy'),
+
+ # Marked as offline for downscaling but not actually scaled down ([DOWNSCALE])
+ self._util_create_node(display_name='prepared_downscale1', assigned_labels=['mxnetlinux-cpu'],
+ nb_executors=1, temporarily_offline=True, is_offline=False,
+ offline_reason=autoscaling.DOWNSCALE_REASON),
+ self._util_create_node(display_name='prepared_downscale2', assigned_labels=['mxnetlinux-cpu'],
+ nb_executors=1, temporarily_offline=True, is_offline=False,
+ offline_reason=autoscaling.DOWNSCALE_MANUAL_REASON + ' Reason2'),
+ self._util_create_node(display_name='prepared_downscale3', assigned_labels=['mxnetlinux-cpu'],
+ nb_executors=1, temporarily_offline=True, is_offline=False,
+ offline_reason=autoscaling.DOWNSCALE_MANUAL_REASON + ' Reason3'),
+
+ # Created but never started instance
+ self._util_create_node(display_name='never_created1', assigned_labels=['mxnetlinux-cpu'],
+ nb_executors=1, temporarily_offline=False, is_offline=True),
+ self._util_create_node(display_name='never_created2', assigned_labels=['mxnetlinux-cpu'],
+ nb_executors=1, temporarily_offline=False, is_offline=True),
+ self._util_create_node(display_name='never_created3', assigned_labels=['mxnetlinux-cpu'],
+ nb_executors=1, temporarily_offline=False, is_offline=True),
+
+ # Unmanaged slave (e.g. physical devices)
+ self._util_create_node(display_name='unmanaged1', assigned_labels=['unmanaged'],
+ nb_executors=1, temporarily_offline=False, is_offline=False),
+ self._util_create_node(display_name='unmanaged2', assigned_labels=['unmanaged'],
+ nb_executors=1, temporarily_offline=False, is_offline=False)
+
+ ]
+
+ instance_uptime = {
+ 'fresh_instance1': 10, 'fresh_instance2': 10, 'fresh_instance3': 10,
+ 'offline_ignore1': 10, 'offline_ignore2': 500000,
+ 'start_timeout1': 50000000, 'start_timeout2': 50000000, 'start_timeout3': 50000000,
+ 'taken_offline_unhealthy1': 10, 'taken_offline_unhealthy2': 500, 'taken_offline_unhealthy3': 50000,
+ 'prepared_downscale1': 10, 'prepared_downscale2': 500, 'prepared_downscale3': 50000,
+ 'orphan1': 10, 'orphan2': 1023901
+ }
+
+ unconnected = {'mxnetlinux-cpu': ['start_timeout1', 'start_timeout2', 'start_timeout3', 'orphan1', 'orphan2']}
+
+ (faulty_nodes, orphaned) = autoscaling._determine_faulty_nodes(
+ nodes=nodes_data,
+ unconnected_instances=unconnected,
+ instance_uptime=instance_uptime)
+
+ count_expected_filtered_nodes = 12
+ self.assertEqual(orphaned, ['orphan1', 'orphan2'])
+ assert len(faulty_nodes.keys()) == 1, \
+ 'Expected mxnetlinux-cpu as only faulty_nodes key, got {}'.format(pprint.pformat(faulty_nodes.keys()))
+ assert len(faulty_nodes['mxnetlinux-cpu']) == count_expected_filtered_nodes, \
+ 'Expected {} filtered nodes, got {}: {}'.format(count_expected_filtered_nodes,
+ len(faulty_nodes['mxnetlinux-cpu']),
+ pprint.pformat(faulty_nodes['mxnetlinux-cpu']))
+
+ def test_merge_dicts_nested_lists(self):
+ # Partial insect
+ assert autoscaling._merge_dicts_nested_lists(
+ dict1={1: ['a', 'b'], 2: ['c', 'd']},
+ dict2={2: ['d', 'e'], 3: ['f', 'g']}
+ ) == {1: ['a', 'b'], 2: ['c', 'd', 'd', 'e'], 3: ['f', 'g']}
+
+ # No intersect
+ assert autoscaling._merge_dicts_nested_lists(
+ dict1={1: ['a', 'b']},
+ dict2={2: ['d', 'e']}
+ ) == {1: ['a', 'b'], 2: ['d', 'e']}
+
+ # Partial null
+ assert autoscaling._merge_dicts_nested_lists(
+ dict1=None,
+ dict2={2: ['d', 'e']}
+ ) == {2: ['d', 'e']}
+
+ # Full null
+ assert autoscaling._merge_dicts_nested_lists(
+ dict1=None,
+ dict2=None
+ ) == {}
+
+ # Empty lists
+ assert autoscaling._merge_dicts_nested_lists(
+ dict1={1: []},
+ dict2={2: []}
+ ) == {1: [], 2: []}
+
+ # Same reference
+ dict_same = {1: ['a', 'b'], 2: ['c', 'd']}
+ assert autoscaling._merge_dicts_nested_lists(
+ dict1=dict_same,
+ dict2=dict_same
+ ) == {1: ['a', 'b', 'a', 'b'], 2: ['c', 'd', 'c', 'd']}
+
+
+if __name__ == '__main__':
+ import nose2
+
+ nose2.main()
diff --git a/services/jenkins-autoscaling/requirements.txt b/services/jenkins-autoscaling/requirements.txt
new file mode 100644
index 0000000..17dc93a
--- /dev/null
+++ b/services/jenkins-autoscaling/requirements.txt
@@ -0,0 +1,30 @@
+backcall==0.1.0
+boto3==1.7.63
+botocore==1.10.63
+certifi==2018.4.16
+chardet==3.0.4
+decorator==4.3.0
+docutils==0.14
+idna==2.7
+ipython==6.4.0
+ipython-genutils==0.2.0
+jedi==0.12.1
+jenkinsapi==0.3.6
+jmespath==0.9.3
+joblib==0.12.1
+parso==0.3.1
+pexpect==4.6.0
+pickleshare==0.7.4
+placebo==0.8.1
+prompt-toolkit==1.0.15
+ptyprocess==0.6.0
+Pygments==2.2.0
+python-dateutil==2.7.3
+pytz==2018.5
+requests==2.19.1
+s3transfer==0.1.13
+simplegeneric==0.8.1
+six==1.11.0
+traitlets==4.3.2
+urllib3==1.23
+wcwidth==0.1.7