You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by da...@apache.org on 2022/11/21 14:22:26 UTC
[beam] branch master updated: [Playground] Use current Go SDK by default (#24256)
This is an automated email from the ASF dual-hosted git repository.
damccorm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new f82aac7ac60 [Playground] Use current Go SDK by default (#24256)
f82aac7ac60 is described below
commit f82aac7ac60141c5030f1311033742ef162942cd
Author: Evgeny Antyshev <ea...@gmail.com>
AuthorDate: Mon Nov 21 17:22:13 2022 +0300
[Playground] Use current Go SDK by default (#24256)
* gradle-cache
* python
* SDK_TAG
* fix
* java
* only python
* write cache
* go
* AIO
* python
* nit
---
.../workflows/playground_examples_ci_reusable.yml | 17 +++++---
playground/backend/containers/go/Dockerfile | 33 ++++++++-------
playground/backend/containers/go/build.gradle | 21 +++++++++-
playground/backend/containers/go/setup_sdk.sh | 47 ++++++++++++++++++++++
playground/backend/containers/python/build.gradle | 10 ++---
5 files changed, 98 insertions(+), 30 deletions(-)
diff --git a/.github/workflows/playground_examples_ci_reusable.yml b/.github/workflows/playground_examples_ci_reusable.yml
index 05c255adfb3..d00ecd3b5bd 100644
--- a/.github/workflows/playground_examples_ci_reusable.yml
+++ b/.github/workflows/playground_examples_ci_reusable.yml
@@ -123,14 +123,17 @@ jobs:
with:
cache-read-only: false
- - name: Build Python SDK runner base image
- if: env.SDK == 'python'
+ - name: Build an SDK runner base image and set SDK_TAG if needed
run: |
set -uex
- ./gradlew -i :sdks:python:container:py37:docker -Pdocker-tag="$DOCKERTAG"
- echo "SDK_TAG=${DOCKERTAG}" >> $GITHUB_ENV
-
- # TODO build from the current ./sdks/... for Java and Go
+ # TODO make this a part of playground:backend:containers:python:docker task
+ if [ "$SDK" == "python" ]; then
+ # builds apache/beam_python3.7_sdk:$DOCKERTAG image
+ ./gradlew -i :sdks:python:container:py37:docker -Pdocker-tag="$DOCKERTAG"
+ # and set SDK_TAG to DOCKERTAG so that the next step would find it
+ echo "SDK_TAG=${DOCKERTAG}" >> $GITHUB_ENV
+ fi
+
- name: Build SDK Backend Docker image
run: |
set -ex
@@ -139,6 +142,8 @@ jobs:
opts="$opts -Psdk-tag=$SDK_TAG"
fi
+ # by default (w/o -Psdk-tag) runner uses BEAM from local ./sdks
+ # TODO Java SDK doesn't, it uses 2.42.0, fix this
./gradlew -i playground:backend:containers:$SDK:docker $opts
- name: Set docker image
diff --git a/playground/backend/containers/go/Dockerfile b/playground/backend/containers/go/Dockerfile
index 575e8f9e121..b52b70a70b9 100644
--- a/playground/backend/containers/go/Dockerfile
+++ b/playground/backend/containers/go/Dockerfile
@@ -16,35 +16,34 @@
# limitations under the License.
###############################################################################
#Dokerfile to set up the Beam Go SDK
-ARG BASE_IMAGE=golang:1.18-bullseye
+ARG BASE_IMAGE
FROM $BASE_IMAGE
+ARG SDK_TAG
+ARG SDK_TAG_LOCAL
# Setup Go Environment
ENV GOPATH /go
ENV PATH $GOPATH/bin:$PATH
-RUN mkdir -p "$GOPATH/src" "$GOPATH/bin" && chmod -R 777 "$GOPATH"
+RUN mkdir -p "$GOPATH/src" "$GOPATH/bin" &&\
+ chmod -R 777 "$GOPATH/bin"
+
+
RUN go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.28.1 &&\
go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.2.0
-# Prepare Application
-COPY src /go/src/playground/backend
+# Setup Go SDK dependencies
+COPY beam /opt/playground/beam
+COPY setup_sdk.sh /
-ARG BEAM_VERSION=2.42.0
-ENV PREPARED_MOD_DIR=/opt/playground/prepared_folder/
-ENV PIPELINES_FOLDER_NAME="executable_files"
-RUN mkdir -p /opt/playground/ $PREPARED_MOD_DIR
-WORKDIR $PREPARED_MOD_DIR
+ENV BEAM_SRC /opt/playground/beam
+ENV PREPARED_MOD_DIR /opt/playground/prepared_folder
+RUN /setup_sdk.sh /opt/playground/prepared_folder
-RUN go mod init $PIPELINES_FOLDER_NAME &&\
- go get github.com/apache/beam/sdks/v2/go/pkg/beam@v$BEAM_VERSION &&\
- go get github.com/apache/beam/sdks/v2/go/pkg/beam/io/filesystem/gcs@v$BEAM_VERSION &&\
- go mod download all
+# Prepare Application
+COPY src /go/src/playground/backend
# Build Application
-
-#RUN
WORKDIR /go/src/playground/backend
-RUN ls
# Build Application
RUN go mod download &&\
go mod tidy &&\
@@ -62,7 +61,7 @@ COPY src/configs /opt/playground/backend/configs/
# Install mitmpoxy
RUN mkdir /opt/mitmproxy &&\
cd /opt/mitmproxy &&\
- wget https://snapshots.mitmproxy.org/7.0.4/mitmproxy-7.0.4-linux.tar.gz &&\
+ wget -q https://snapshots.mitmproxy.org/7.0.4/mitmproxy-7.0.4-linux.tar.gz &&\
tar -zxvf mitmproxy-7.0.4-linux.tar.gz &&\
mkdir /usr/local/share/ca-certificates/extra
COPY allow_list_proxy.py /opt/mitmproxy/
diff --git a/playground/backend/containers/go/build.gradle b/playground/backend/containers/go/build.gradle
index 3e9439e4be2..3a47d930d77 100644
--- a/playground/backend/containers/go/build.gradle
+++ b/playground/backend/containers/go/build.gradle
@@ -43,6 +43,10 @@ task copyDockerfileDependencies(type: Copy) {
from 'entrypoint.sh'
into 'build/'
}
+ copy {
+ from 'setup_sdk.sh'
+ into 'build/'
+ }
copy {
from '../../../infrastructure/proxy/allow_list.py'
into 'build/'
@@ -55,6 +59,18 @@ task copyDockerfileDependencies(type: Copy) {
from '../../../playground'
into 'build/playground'
}
+ copy {
+ from '../../../../sdks/go'
+ into 'build/beam/go'
+ }
+ copy {
+ from '../../../../sdks/go.mod'
+ into 'build/beam'
+ }
+ copy {
+ from '../../../../sdks/go.sum'
+ into 'build/beam'
+ }
}
docker {
@@ -67,7 +83,10 @@ docker {
tags containerImageTags()
buildArgs(['BASE_IMAGE': project.rootProject.hasProperty(["base-image"]) ?
project.rootProject["base-image"] :
- "golang:1.18-bullseye" ])
+ "golang:1.18-bullseye",
+ 'SDK_TAG': project.rootProject.hasProperty(["sdk-tag"]) ?
+ project.rootProject["sdk-tag"] : project.rootProject.sdk_version,
+ 'SDK_TAG_LOCAL': project.rootProject.sdk_version])
}
// Ensure that we build the required resources and copy and file dependencies from related projects
diff --git a/playground/backend/containers/go/setup_sdk.sh b/playground/backend/containers/go/setup_sdk.sh
new file mode 100755
index 00000000000..20ba227c19e
--- /dev/null
+++ b/playground/backend/containers/go/setup_sdk.sh
@@ -0,0 +1,47 @@
+#!/bin/bash -uex
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Usage: ./setup_sdk.sh /opt/playground/prepared_folder
+# Setup Go Beam SDK dev environment, so that examples' dependencies were met.
+# Consumed envs:
+# - PREPARED_MOD_DIR: result module path
+# - SDK_TAG: desired SDK version, for ex. "2.42.0", "2.44.0dev"
+# - SDK_TAG_LOCAL: special value for SDK_TAG for the local Go SDK (defined by build system)
+# - BEAM_SRC: Go SDK location
+
+PIPELINES_MODULE="executable_files"
+
+# Project convention for local versions 2.44.0.dev is incorrect in Go
+# Replace this with 2.44.0-dev
+SDK_TAG=${SDK_TAG/%.dev/-dev}
+SDK_TAG_LOCAL=${SDK_TAG_LOCAL/%.dev/-dev}
+
+mkdir -p $PREPARED_MOD_DIR
+cd $PREPARED_MOD_DIR
+
+BEAM_PKG=github.com/apache/beam/sdks/v2
+
+go mod init $PIPELINES_MODULE
+go mod edit -require=$BEAM_PKG@v$SDK_TAG
+
+if [ "$SDK_TAG" == "$SDK_TAG_LOCAL" ]; then
+ go mod edit -replace=$BEAM_PKG@v$SDK_TAG=$BEAM_SRC
+fi
+
+go get -x $BEAM_PKG/go/pkg/beam@v$SDK_TAG
+go get -x $BEAM_PKG/go/test/integration@v$SDK_TAG
+
+go mod download -x all
diff --git a/playground/backend/containers/python/build.gradle b/playground/backend/containers/python/build.gradle
index d12a15c2fb2..a1b6688ae52 100644
--- a/playground/backend/containers/python/build.gradle
+++ b/playground/backend/containers/python/build.gradle
@@ -69,14 +69,12 @@ docker {
project.docker_image_default_repo_root)
files "./build/"
tags containerImageTags()
- buildArgs(['BASE_IMAGE': project.rootProject.hasProperty(["base-image"]) ?
- project.rootProject["base-image"] :
- "apache/beam_python3.7_sdk" ])
buildArgs(['GO_BASE_IMAGE': project.rootProject.hasProperty(["go-base-image"]) ?
project.rootProject["go-base-image"] :
- "golang:1.18-bullseye" ])
- buildArgs(['SDK_TAG': project.rootProject.hasProperty(["sdk-tag"]) ?
- project.rootProject["sdk-tag"] : project.rootProject.sdk_version])
+ "golang:1.18-bullseye",
+ 'SDK_TAG': project.rootProject.hasProperty(["sdk-tag"]) ?
+ project.rootProject["sdk-tag"] : project.rootProject.sdk_version
+ ])
}
// Ensure that we build the required resources and copy and file dependencies from related projects