You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by sh...@apache.org on 2026/09/17 10:01:13 UTC
(airflow-ci-infra) 32/34: switch to Autoscaling Runner Scale Sets mode
This is an automated email from the ASF dual-hosted git repository.
shahar1 pushed a commit to branch gha-arc-rebase
in repository https://gitbox.apache.org/repos/asf/airflow-ci-infra.git
commit c4e8315489c19f7d94c96bf9e437d99c1ec8e354
Author: hussein-awala <hu...@awala.fr>
AuthorDate: Fri Aug 30 01:28:39 2024 +0200
switch to Autoscaling Runner Scale Sets mode
---
README.rst | 2 +-
helm/{ci-helmfile.yaml => ci.yaml} | 22 ++--
helm/infra-helmfile.yaml | 9 ++
helm/runners-config/templates/runners.yaml | 71 -------------
helm/runners-config/values.yaml | 117 ---------------------
.../Chart.yaml => values/ebs-csi-dirver.yaml} | 10 +-
...r.yaml => gha-runner-scale-set-controller.yaml} | 4 +-
.../gha-runner-scale-sets/common.yaml.gotmpl | 61 +++++++++++
.../runners.yaml} | 42 ++++++--
runner/Dockerfile | 52 +++++----
terraform/eks/eks.tf | 9 +-
terraform/eks/iam.tf | 44 +++++++-
terraform/eks/variables.tf | 17 +--
13 files changed, 212 insertions(+), 248 deletions(-)
diff --git a/README.rst b/README.rst
index aa47573..12fa8ef 100644
--- a/README.rst
+++ b/README.rst
@@ -52,7 +52,7 @@ you need to follow the steps below:
3. Update kubeconfig to access the EKS cluster::
- AWS_PROFILE=airflow aws eks --region us-east-2 update-kubeconfig --name airflow
+ AWS_PROFILE=airflow aws eks --region us-east-2 update-kubeconfig --name airflow --alias airflow
This command will update the kubeconfig file located at `~/.kube/config` to include the EKS cluster.
You can verify that the cluster is added by running `kubectl get nodes`.
diff --git a/helm/ci-helmfile.yaml b/helm/ci.yaml
similarity index 62%
rename from helm/ci-helmfile.yaml
rename to helm/ci.yaml
index eda1b4b..e1da5a0 100644
--- a/helm/ci-helmfile.yaml
+++ b/helm/ci.yaml
@@ -15,18 +15,24 @@
# specific language governing permissions and limitations
# under the License.
---
-repositories:
- - name: actions-runner-controller
- url: https://actions-runner-controller.github.io/actions-runner-controller
+environments:
+ default:
+ values:
+ - ./values/gha-runner-scale-sets/runners.yaml
releases:
- name: actions-runner-controller
- chart: actions-runner-controller/actions-runner-controller
+ chart: oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set-controller
namespace: ci
- version: 0.23.7
+ version: 0.9.3
values:
- - ./values/actions-runner-controller.yaml
+ - ./values/gha-runner-scale-set-controller.yaml
- - name: runners-config
- chart: ./runners-config
+ {{- range $key, $value := .Values.runnerScaleSets }}
+ - name: {{ $key }}
+ chart: oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set
namespace: ci
+ version: 0.9.3
+ values:
+ - ./values/gha-runner-scale-sets/common.yaml.gotmpl
+ {{- end }}
diff --git a/helm/infra-helmfile.yaml b/helm/infra-helmfile.yaml
index f69bc41..d8c49e5 100644
--- a/helm/infra-helmfile.yaml
+++ b/helm/infra-helmfile.yaml
@@ -20,12 +20,21 @@ repositories:
url: https://charts.jetstack.io
- name: autoscaler
url: https://kubernetes.github.io/autoscaler
+ - name: aws-ebs-csi-driver
+ url: https://kubernetes-sigs.github.io/aws-ebs-csi-driver
releases:
- name: cluster-config
chart: ./cluster-config
namespace: kube-system
+ # https://github.com/kubernetes-sigs/aws-ebs-csi-driver/tree/master/charts/aws-ebs-csi-driver
+ - name: aws-ebs-csi-driver
+ chart: aws-ebs-csi-driver/aws-ebs-csi-driver
+ namespace: kube-system
+ values:
+ - ./values/ebs-csi-dirver.yaml
+
# https://cert-manager.io/docs/installation/helm/
- name: cert-manager
chart: jetstack/cert-manager
diff --git a/helm/runners-config/templates/runners.yaml b/helm/runners-config/templates/runners.yaml
deleted file mode 100644
index b11257f..0000000
--- a/helm/runners-config/templates/runners.yaml
+++ /dev/null
@@ -1,71 +0,0 @@
-{{/*
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-*/}}
-
-################################################################################
-# Airflow GitHub Actions Runner Deployments and HRAutoscaler
-################################################################################
-{{ $common := .Values.common }}
-{{ $values := .Values }}
-{{ range $runner := $values.runners }}
----
-apiVersion: actions.summerwind.dev/v1alpha1
-kind: RunnerDeployment
-metadata:
- name: {{ $runner.name }}-runners-deployment
-spec:
- template:
- spec:
- repository: {{ $values.githubRepository }}
- nodeSelector:
- {{- $runner.nodeSelector | toYaml | nindent 8 }}
- tolerations:
- {{- $runner.tolerations | toYaml | nindent 8 }}
- labels:
- {{- $common.runnersLabels | toYaml | nindent 8 }}
- {{- $runner.labels | toYaml | nindent 8 }}
- affinity:
- podAntiAffinity:
- requiredDuringSchedulingIgnoredDuringExecution:
- - labelSelector:
- matchExpressions:
- - key: runner-deployment-name
- operator: In
- values:
- - {{ $runner.name }}-runners-deployment
- topologyKey: kubernetes.io/hostname
- env:
- - name: DISABLE_RUNNER_UPDATE
- value: "false" # TODO: Set to true to disable runner update after patching the latest version
----
-apiVersion: actions.summerwind.dev/v1alpha1
-kind: HorizontalRunnerAutoscaler
-metadata:
- name: {{ $runner.name }}-runners-deployment-autoscaler
-spec:
- scaleDownDelaySecondsAfterScaleOut: {{ $values.scaleDownDelaySecondsAfterScaleOut }}
- scaleTargetRef:
- kind: RunnerDeployment
- name: {{ $runner.name }}-runners-deployment
- minReplicas: {{ $runner.minReplicas }}
- maxReplicas: {{ $runner.maxReplicas }}
- metrics:
- - type: TotalNumberOfQueuedAndInProgressWorkflowRuns
- repositoryNames:
- - {{ $values.githubRepository }}
-{{ end }}
diff --git a/helm/runners-config/values.yaml b/helm/runners-config/values.yaml
deleted file mode 100644
index e22ed43..0000000
--- a/helm/runners-config/values.yaml
+++ /dev/null
@@ -1,117 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
----
-githubRepository: apache/airflow
-
-scaleDownDelaySecondsAfterScaleOut: 30
-
-common:
- runnersLabels:
- - airflow-runner
- - eks-runner
-
-runners:
- - name: small
- minReplicas: 0
- maxReplicas: 30
- labels:
- - small
- - arm64
- nodeSelector:
- node-type: gha-runners
- size: small
- arch: arm64
- tolerations:
- - key: node-type
- operator: Equal
- value: gha-runners
- effect: NoSchedule
- - name: medium
- minReplicas: 0
- maxReplicas: 30
- labels:
- - medium
- - arm64
- nodeSelector:
- node-type: gha-runners
- size: medium
- arch: arm64
- tolerations:
- - key: node-type
- operator: Equal
- value: gha-runners
- effect: NoSchedule
- - name: large
- minReplicas: 0
- maxReplicas: 30
- labels:
- - large
- - arm64
- nodeSelector:
- node-type: gha-runners
- size: large
- arch: arm64
- tolerations:
- - key: node-type
- operator: Equal
- value: gha-runners
- effect: NoSchedule
- - name: small-x64
- minReplicas: 0
- maxReplicas: 30
- labels:
- - small
- - X64
- nodeSelector:
- node-type: gha-runners
- size: small
- arch: x64
- tolerations:
- - key: node-type
- operator: Equal
- value: gha-runners
- effect: NoSchedule
- - name: medium-x64
- minReplicas: 0
- maxReplicas: 30
- labels:
- - medium
- - X64
- nodeSelector:
- node-type: gha-runners
- size: medium
- arch: x64
- tolerations:
- - key: node-type
- operator: Equal
- value: gha-runners
- effect: NoSchedule
- - name: large-x64
- minReplicas: 0
- maxReplicas: 30
- labels:
- - large
- - X64
- nodeSelector:
- node-type: gha-runners
- size: large
- arch: x64
- tolerations:
- - key: node-type
- operator: Equal
- value: gha-runners
- effect: NoSchedule
diff --git a/helm/runners-config/Chart.yaml b/helm/values/ebs-csi-dirver.yaml
similarity index 84%
rename from helm/runners-config/Chart.yaml
rename to helm/values/ebs-csi-dirver.yaml
index 7113941..7f8f0d0 100644
--- a/helm/runners-config/Chart.yaml
+++ b/helm/values/ebs-csi-dirver.yaml
@@ -15,9 +15,7 @@
# specific language governing permissions and limitations
# under the License.
---
-apiVersion: v2
-name: runners-config
-description: A Helm chart to manage the ARC runners
-type: application
-version: 0.1.0
-appVersion: "1.16.0"
+controller:
+ serviceAccount:
+ annotations:
+ eks.amazonaws.com/role-arn: arn:aws:iam::827901512104:role/ebs-csi-driver
diff --git a/helm/values/actions-runner-controller.yaml b/helm/values/gha-runner-scale-set-controller.yaml
similarity index 94%
copy from helm/values/actions-runner-controller.yaml
copy to helm/values/gha-runner-scale-set-controller.yaml
index bb5786f..63ab994 100644
--- a/helm/values/actions-runner-controller.yaml
+++ b/helm/values/gha-runner-scale-set-controller.yaml
@@ -19,9 +19,7 @@ replicaCount: 1
labels:
type: ci
- app: actions-runner-controller
-
-syncPeriod: 10s
+ app: gha-runner-scale-set-controller
nodeSelector:
node-type: default
diff --git a/helm/values/gha-runner-scale-sets/common.yaml.gotmpl b/helm/values/gha-runner-scale-sets/common.yaml.gotmpl
new file mode 100644
index 0000000..9ed5238
--- /dev/null
+++ b/helm/values/gha-runner-scale-sets/common.yaml.gotmpl
@@ -0,0 +1,61 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+---
+{{ $config := index .Values.runnerScaleSets .Release.Name }}
+
+githubConfigUrl: https://github.com/apache
+
+githubConfigSecret: github-config
+
+runnerGroup: airflow-ci
+
+minRunners: {{ $config.minRunners }}
+
+maxRunners: {{ $config.maxRunners }}
+
+containerMode:
+ type: "dind"
+
+template:
+ spec:
+ containers:
+ - name: runner
+ image: {{ .Values.image }}
+ command: [ "/home/runner/run.sh" ]
+ nodeSelector:
+ node-type: gha-runners
+ size: {{ $config.size }}
+ arch: {{ $config.arch }}
+ tolerations:
+ - key: node-type
+ operator: Equal
+ value: gha-runners
+ effect: NoSchedule
+ affinity:
+ podAntiAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ - labelSelector:
+ matchExpressions:
+ - key: actions.github.com/scale-set-name
+ operator: In
+ values:
+ - {{ .Release.Name }}
+ topologyKey: kubernetes.io/hostname
+
+controllerServiceAccount:
+ name: actions-runner-controller-gha-rs-controller
+ namespace: {{ .Release.Namespace }}
diff --git a/helm/values/actions-runner-controller.yaml b/helm/values/gha-runner-scale-sets/runners.yaml
similarity index 52%
rename from helm/values/actions-runner-controller.yaml
rename to helm/values/gha-runner-scale-sets/runners.yaml
index bb5786f..40268eb 100644
--- a/helm/values/actions-runner-controller.yaml
+++ b/helm/values/gha-runner-scale-sets/runners.yaml
@@ -15,13 +15,35 @@
# specific language governing permissions and limitations
# under the License.
---
-replicaCount: 1
-
-labels:
- type: ci
- app: actions-runner-controller
-
-syncPeriod: 10s
-
-nodeSelector:
- node-type: default
+image: ghcr.io/apache/airflow-ci-infra/actions-runner:20240830-rc1
+runnerScaleSets:
+ arc-small-amd:
+ minRunners: 0
+ maxRunners: 30
+ size: small
+ arch: x64
+ arc-medium-amd:
+ minRunners: 0
+ maxRunners: 30
+ size: medium
+ arch: x64
+ arc-large-amd:
+ minRunners: 0
+ maxRunners: 30
+ size: large
+ arch: x64
+ arc-small-arm:
+ minRunners: 0
+ maxRunners: 30
+ size: small
+ arch: arm64
+ arc-medium-arm:
+ minRunners: 0
+ maxRunners: 30
+ size: medium
+ arch: arm64
+ arc-large-arm:
+ minRunners: 0
+ maxRunners: 30
+ size: large
+ arch: arm64
diff --git a/runner/Dockerfile b/runner/Dockerfile
index d1ee441..05daa82 100644
--- a/runner/Dockerfile
+++ b/runner/Dockerfile
@@ -1,27 +1,35 @@
-ARG ACTIONS_RUNNER_VERSION=v2.309.0-ubuntu-22.04-ead26ab
-
-FROM summerwind/actions-runner:${ACTIONS_RUNNER_VERSION}
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+FROM ghcr.io/actions/actions-runner:latest
USER root
-ARG AIRFLOW_RUNNER_VERSION=2.309.0-airflow11
-
-RUN rm -rf $RUNNER_ASSETS_DIR/* \
- && export ARCH=$(arch) \
- && if [ "$ARCH" = "amd64" ] || [ "$ARCH" = "x86_64" ] || [ "$ARCH" = "i386" ]; then export ARCH=x64 ; fi \
- && if [ "$ARCH" = "aarch64" ]; then export ARCH=arm64 ; fi \
- && cd "$RUNNER_ASSETS_DIR" \
- && curl -fLo runner.tar.gz "https://github.com/ashb/runner/releases/download/v${AIRFLOW_RUNNER_VERSION}/actions-runner-linux-${ARCH}-${AIRFLOW_RUNNER_VERSION}.tar.gz" \
- && tar xzf ./runner.tar.gz \
- && rm runner.tar.gz \
- && ./bin/installdependencies.sh \
- && mv ./externals ./externalstmp
-
-# TODO: Remove this once https://github.com/actions/setup-python/issues/705 is solved
-RUN apt-get update && \
- apt-get install -y --no-install-recommends \
- curl nodejs wget unzip vim git jq build-essential libssl-dev libffi-dev python3 python3-venv python3-dev python3-pip python-is-python3 \
- && chown -R runner:runner /usr/local/lib/ --recursive \
- && chown -R runner:runner /usr/local/bin/ --recursive
+RUN apt-get update \
+ && apt-get install -y --no-install-recommends \
+ ca-certificates curl nodejs npm wget unzip vim git jq build-essential netcat \
+ libssl-dev libffi-dev python3 python3-venv python3-dev python3-pip python-is-python3 \
+ && npm install -g yarn \
+ && install -m 0755 -d /etc/apt/keyrings \
+ && curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc \
+ && chmod a+r /etc/apt/keyrings/docker.asc \
+ && echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \
+ $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null \
+ && apt-get update \
+ && apt-get install -y docker-compose-plugin
USER runner
\ No newline at end of file
diff --git a/terraform/eks/eks.tf b/terraform/eks/eks.tf
index adf6fdb..5b4203a 100644
--- a/terraform/eks/eks.tf
+++ b/terraform/eks/eks.tf
@@ -45,7 +45,14 @@ module "eks" {
default_nodes = {
name = "default"
- instance_types = ["t4g.medium"]
+ instance_types = [
+ "t4g.medium",
+ "t4g.large",
+ "t4g.xlarge",
+ "m7g.medium",
+ "m7g.large",
+ "m7g.xlarge",
+ ]
min_size = 0
max_size = 5
diff --git a/terraform/eks/iam.tf b/terraform/eks/iam.tf
index 983ee0e..e2c3000 100644
--- a/terraform/eks/iam.tf
+++ b/terraform/eks/iam.tf
@@ -89,4 +89,46 @@ resource "aws_iam_role" "autoscaler_role" {
resource "aws_iam_role_policy_attachment" "autoscaler_policy_attachment" {
role = aws_iam_role.autoscaler_role.name
policy_arn = aws_iam_policy.autoscaler_policy.arn
-}
\ No newline at end of file
+}
+
+data "aws_iam_policy_document" "ebs_csi_driver_assume_policy_document" {
+ statement {
+ actions = [
+ "sts:AssumeRoleWithWebIdentity"
+ ]
+
+ principals {
+ type = "Federated"
+ identifiers = [
+ module.eks.oidc_provider_arn
+ ]
+ }
+
+ condition {
+ test = "StringEquals"
+ variable = "${module.eks.oidc_provider}:sub"
+ values = [
+ "system:serviceaccount:kube-system:ebs-csi-controller-sa"
+ ]
+ }
+
+ condition {
+ test = "StringEquals"
+ variable = "${module.eks.oidc_provider}:aud"
+ values = [
+ "sts.amazonaws.com"
+ ]
+ }
+
+ }
+}
+
+resource "aws_iam_role" "ebs_csi_driver" {
+ name = "ebs-csi-driver"
+ assume_role_policy = data.aws_iam_policy_document.ebs_csi_driver_assume_policy_document.json
+}
+
+resource "aws_iam_role_policy_attachment" "ebs_csi_driver" {
+ role = aws_iam_role.ebs_csi_driver.name
+ policy_arn = "arn:aws:iam::aws:policy/service-role/AmazonEBSCSIDriverPolicy"
+}
diff --git a/terraform/eks/variables.tf b/terraform/eks/variables.tf
index d04ce6b..045c496 100644
--- a/terraform/eks/variables.tf
+++ b/terraform/eks/variables.tf
@@ -24,20 +24,21 @@ variable "region" {
variable "runners_node_types" {
description = "Node type for the runners"
type = list(string)
- default = ["t4g"]
+ default = [
+ "t4g",
+ "m7g",
+ "m6g",
+ ]
}
variable "x64_runners_node_types" {
description = "X64 node type for the runners"
type = list(string)
default = [
- "r6a",
- "r5",
- "r5ad",
- "r5n",
- "r5b",
- "r5a",
- "r6i",
+ "m7a",
+ "m6a",
+ "t3",
+ "t3a",
]
}