You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by da...@apache.org on 2023/06/26 14:29:17 UTC
[beam] branch master updated: runner image and arc changes (#27118)
This is an automated email from the ASF dual-hosted git repository.
damccorm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new a4281d8f35c runner image and arc changes (#27118)
a4281d8f35c is described below
commit a4281d8f35cf26bc62444f06b48d0019085e8e64
Author: Vlado Djerek <20...@users.noreply.github.com>
AuthorDate: Mon Jun 26 16:29:09 2023 +0200
runner image and arc changes (#27118)
* runner image and arc changes
* add state bucket comment in env file for reference
* add readme for manual image push and sync beam environment
---------
Co-authored-by: vdjerek <Vlado Djerek>
---
.../arc/config/arc_deployment.tpl | 6 +--
.../arc/environments/beam.env | 11 ++---
.github/gh-actions-self-hosted-runners/arc/gke.tf | 1 -
.github/gh-actions-self-hosted-runners/arc/iam.tf | 7 ---
.../arc/images/Dockerfile | 52 ++++++++++++++++++++++
.../arc/images/README.md | 36 +++++++++++++++
.github/workflows/build_runner_image.yml | 49 ++++++++++++++++++++
7 files changed, 146 insertions(+), 16 deletions(-)
diff --git a/.github/gh-actions-self-hosted-runners/arc/config/arc_deployment.tpl b/.github/gh-actions-self-hosted-runners/arc/config/arc_deployment.tpl
index 98eeb8544f7..fd3803d522d 100644
--- a/.github/gh-actions-self-hosted-runners/arc/config/arc_deployment.tpl
+++ b/.github/gh-actions-self-hosted-runners/arc/config/arc_deployment.tpl
@@ -31,9 +31,9 @@ spec:
- "self-hosted"
env: []
resources:
- limits:
- cpu: "4.0"
- memory: "8Gi"
+# limits:
+# cpu: "4.0"
+# memory: "8Gi"
requests:
cpu: "500m"
memory: "500Mi"
diff --git a/.github/gh-actions-self-hosted-runners/arc/environments/beam.env b/.github/gh-actions-self-hosted-runners/arc/environments/beam.env
index e215e4c7bd2..91b336ad10b 100644
--- a/.github/gh-actions-self-hosted-runners/arc/environments/beam.env
+++ b/.github/gh-actions-self-hosted-runners/arc/environments/beam.env
@@ -23,15 +23,16 @@ zone = "us-west1-b"
min_main_node_count = "1"
max_main_node_count = "5"
environment = "beam"
-ingress_domain = "runners.example.com"
+ingress_domain = "action.beam.apache.org"
organization = "apache"
repository = "beam"
github_app_id_secret_name = "gh-app_id"
github_app_install_id_secret_name = "gh-app_installation_id"
github_private_key_secret_name = "gh-pem_key"
-deploy_webhook = "false"
-max_main_replicas = "40"
+deploy_webhook = "true"
+max_main_replicas = "50"
min_main_replicas = "5"
-webhook_scaling = "false"
+webhook_scaling = "true"
runner_group = "beam"
-machine_type = "e2-standard-16"
\ No newline at end of file
+machine_type = "e2-standard-16"
+#state_bucket_name = "beam-arc-state"
\ No newline at end of file
diff --git a/.github/gh-actions-self-hosted-runners/arc/gke.tf b/.github/gh-actions-self-hosted-runners/arc/gke.tf
index d4740b834eb..4bf6f6c5a99 100644
--- a/.github/gh-actions-self-hosted-runners/arc/gke.tf
+++ b/.github/gh-actions-self-hosted-runners/arc/gke.tf
@@ -40,7 +40,6 @@ resource "google_container_node_pool" "actions-runner-pool" {
}
node_config {
machine_type = var.machine_type
- service_account = google_service_account.actions_service_account.email
oauth_scopes = [
"https://www.googleapis.com/auth/cloud-platform"
]
diff --git a/.github/gh-actions-self-hosted-runners/arc/iam.tf b/.github/gh-actions-self-hosted-runners/arc/iam.tf
index 54f209e535b..e6ba2be6545 100644
--- a/.github/gh-actions-self-hosted-runners/arc/iam.tf
+++ b/.github/gh-actions-self-hosted-runners/arc/iam.tf
@@ -16,13 +16,6 @@
# specific language governing permissions and limitations
# under the License.
#
-
-resource "google_service_account" "actions_service_account" {
- account_id = "${var.environment}-runner-gke-sa"
- display_name = "${var.environment}-runner-gke-sa"
-}
-
-
data "google_client_config" "provider" {}
data "google_client_openid_userinfo" "provider_identity" {
diff --git a/.github/gh-actions-self-hosted-runners/arc/images/Dockerfile b/.github/gh-actions-self-hosted-runners/arc/images/Dockerfile
new file mode 100644
index 00000000000..d037d2518f2
--- /dev/null
+++ b/.github/gh-actions-self-hosted-runners/arc/images/Dockerfile
@@ -0,0 +1,52 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+
+FROM ghcr.io/actions-runner-controller/actions-runner-controller/actions-runner:ubuntu-20.04
+
+USER root
+#Install Node
+RUN curl -OL https://nodejs.org/dist/v18.16.0/node-v18.16.0-linux-x64.tar.xz && \
+ tar -C /usr/local -xf node-v18.16.0-linux-x64.tar.xz && \
+ rm node-v18.16.0-linux-x64.tar.xz && \
+ mv /usr/local/node-v18.16.0-linux-x64 /usr/local/node
+ENV PATH="${PATH}:/usr/local/node/bin"
+#Install Go
+ARG go_version=1.20.4
+RUN curl -OL https://go.dev/dl/go${go_version}.linux-amd64.tar.gz && \
+ tar -C /usr/local -xzf go${go_version}.linux-amd64.tar.gz && \
+ rm go${go_version}.linux-amd64.tar.gz
+ENV PATH="${PATH}:/usr/local/go/bin"
+#Install Java
+RUN curl -OL https://cdn.azul.com/zulu/bin/zulu8.70.0.23-ca-jdk8.0.372-linux_x64.tar.gz && \
+ tar -C /usr/local -xzf zulu8.70.0.23-ca-jdk8.0.372-linux_x64.tar.gz && \
+ rm zulu8.70.0.23-ca-jdk8.0.372-linux_x64.tar.gz && \
+ mv /usr/local/zulu8.70.0.23-ca-jdk8.0.372-linux_x64 /usr/local/java
+ENV PATH="${PATH}:/usr/local/java/bin"
+#Install Gradle
+RUN curl -OL https://services.gradle.org/distributions/gradle-7.3.3-bin.zip && \
+ unzip gradle-7.3.3-bin.zip && \
+ rm gradle-7.3.3-bin.zip && \
+ mv gradle-7.3.3 /usr/local/gradle
+ENV PATH="${PATH}:/usr/local/gradle/bin"
+
+
+# Needed to transfer path addtitions to runner environment
+RUN echo PATH=$PATH >> /runnertmp/.env
+USER runner
\ No newline at end of file
diff --git a/.github/gh-actions-self-hosted-runners/arc/images/README.md b/.github/gh-actions-self-hosted-runners/arc/images/README.md
new file mode 100644
index 00000000000..6db908d1bc4
--- /dev/null
+++ b/.github/gh-actions-self-hosted-runners/arc/images/README.md
@@ -0,0 +1,36 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+# Manual build and push
+
+First set a tag you want to use:
+```
+export RUNNER_IMAGE_TAG=some_tag
+```
+After which you run the build command:
+```
+docker build -t us-central1-docker.pkg.dev/apache-beam-testing/beam-github-actions/beam-arc-runner:$RUNNER_IMAGE_TAG -t us-central1-docker.pkg.dev/apache-beam-testing/beam-github-actions/beam-arc-runner:$(git rev-parse --short HEAD) .
+```
+This builds and tags the image with both the Git SHA and desired tag set.
+
+Authenticate to the docker repository in GCP with:
+```
+gcloud auth configure-docker us-central1-docker.pkg.dev
+```
+docker push us-central1-docker.pkg.dev/apache-beam-testing/beam-github-actions/beam-arc-runner:$RUNNER_IMAGE_TAG
+docker push us-central1-docker.pkg.dev/apache-beam-testing/beam-github-actions/beam-arc-runner:$(git rev-parse --short HEAD)
+```
\ No newline at end of file
diff --git a/.github/workflows/build_runner_image.yml b/.github/workflows/build_runner_image.yml
new file mode 100644
index 00000000000..cdd2b34e3ce
--- /dev/null
+++ b/.github/workflows/build_runner_image.yml
@@ -0,0 +1,49 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+name: Build and Version Runner Docker Image
+
+on:
+ pull_request_target:
+ branches: ['master']
+ paths: ['.github/gh-actions-self-hosted-runners/arc/images/**']
+
+jobs:
+ build-and-version-runner:
+ env:
+ working-directory: .github/gh-actions-self-hosted-runners/arc/images/
+ runs-on: [self-hosted, ubuntu-20.04]
+
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v2
+ with:
+ ref: ${{ github.event.pull_request.head.sha }}
+
+ - name: Set up Docker Buildx
+ uses: docker/setup-buildx-action@v1
+
+ - name: Build and push Docker image
+ id: docker_build
+ uses: docker/build-push-action@v2
+ with:
+ context: ${{ env.working-directory }}
+ push: false
+ tags: |
+ us-central1-docker.pkg.dev/apache-beam-testing/beam-github-actions/beam-arc-runner:latest
+ us-central1-docker.pkg.dev/apache-beam-testing/beam-github-actions/beam-arc-runner:${{ github.sha }}