You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by da...@apache.org on 2023/02/01 19:56:10 UTC
[beam] branch master updated: [Tour of Beam] [Task] Fix dependency management for 2.44 Playground java runner (#25230)
This is an automated email from the ASF dual-hosted git repository.
damccorm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 067e4db747b [Tour of Beam] [Task] Fix dependency management for 2.44 Playground java runner (#25230)
067e4db747b is described below
commit 067e4db747b04adf517f12d951176827d55b4b09
Author: Oleh Borysevych <ol...@akvelon.com>
AuthorDate: Wed Feb 1 21:55:57 2023 +0200
[Tour of Beam] [Task] Fix dependency management for 2.44 Playground java runner (#25230)
* fixing dependency management for java
* use correct kafka client
* revert unneeded changes
* minor formatting
* code review comment
* extract version to variable
---
.github/workflows/build_playground_backend.yml | 9 ----
.github/workflows/playground_deploy_backend.yml | 9 ----
.../workflows/playground_examples_ci_reusable.yml | 6 +--
playground/backend/containers/java/Dockerfile | 55 +++++++++-------------
playground/backend/containers/java/build.gradle | 8 ++--
5 files changed, 29 insertions(+), 58 deletions(-)
diff --git a/.github/workflows/build_playground_backend.yml b/.github/workflows/build_playground_backend.yml
index 0eff0e4f370..137fe3cf6ab 100644
--- a/.github/workflows/build_playground_backend.yml
+++ b/.github/workflows/build_playground_backend.yml
@@ -68,15 +68,6 @@ jobs:
- name: Run PreCommit
run: ./gradlew playground:backend:precommit
- - name: install npm
- uses: actions/setup-node@v3
- with:
- node-version: '14'
- - name: install docker linter
- run: npm install -g dockerlint
- - name: lint dockerfile
- run: dockerlint Dockerfile
- working-directory: playground/backend/containers/java
- uses: hashicorp/setup-terraform@v1
with:
terraform_version: ${{ env.TERRAFORM_VERSION }}
diff --git a/.github/workflows/playground_deploy_backend.yml b/.github/workflows/playground_deploy_backend.yml
index 147e4b781f7..b42db802778 100644
--- a/.github/workflows/playground_deploy_backend.yml
+++ b/.github/workflows/playground_deploy_backend.yml
@@ -61,15 +61,6 @@ jobs:
- name: Run PreCommit
run: ./gradlew playground:backend:precommit
- - name: install npm
- uses: actions/setup-node@v3
- with:
- node-version: '14'
- - name: install docker linter
- run: npm install -g dockerlint
- - name: lint dockerfile
- run: dockerlint Dockerfile
- working-directory: playground/backend/containers/java
- uses: hashicorp/setup-terraform@v1
with:
terraform_version: ${{ env.TERRAFORM_VERSION }}
diff --git a/.github/workflows/playground_examples_ci_reusable.yml b/.github/workflows/playground_examples_ci_reusable.yml
index 7c5f039c348..f3f6417f597 100644
--- a/.github/workflows/playground_examples_ci_reusable.yml
+++ b/.github/workflows/playground_examples_ci_reusable.yml
@@ -34,7 +34,7 @@ on:
type: string
required: true
env:
- BEAM_VERSION: 2.43.0
+ BEAM_VERSION: 2.44.0
jobs:
check_has_examples:
name: pre-check
@@ -154,11 +154,11 @@ jobs:
fi
if [ "$SDK" == "java" ]; then
# Java uses a fixed BEAM_VERSION
- opts="$opts -Pbase-image=apache/beam_java8_sdk:$BEAM_VERSION"
+ opts="$opts -Psdk-tag=$BEAM_VERSION"
fi
# by default (w/o -Psdk-tag) runner uses BEAM from local ./sdks
- # TODO Java SDK doesn't, it uses 2.42.0, fix this
+ # TODO Java SDK doesn't, it uses 2.44.0, fix this
./gradlew -i playground:backend:containers:$SDK:docker $opts
- name: Set docker image
diff --git a/playground/backend/containers/java/Dockerfile b/playground/backend/containers/java/Dockerfile
index ca13c38f0c4..2a1561c3b0b 100644
--- a/playground/backend/containers/java/Dockerfile
+++ b/playground/backend/containers/java/Dockerfile
@@ -15,10 +15,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
###############################################################################
-ARG BEAM_VERSION=2.43.0
-ARG BASE_IMAGE=apache/beam_java8_sdk:${BEAM_VERSION}
+ARG BEAM_VERSION=2.44.0
FROM golang:1.18-bullseye AS build
-
+ARG BEAM_VERSION
# Setup Go Environment
ENV GOPATH /go
ENV PATH $GOPATH/bin:$PATH
@@ -37,15 +36,25 @@ RUN go mod download &&\
cd cmd/server &&\
go build -o /go/bin/server_java_backend
-FROM $BASE_IMAGE
-# docker linter we use doesn't allow to have ARG declaration without value although it's a valid syntax
-ARG BEAM_VERSION=2.42.0
-ARG HAMCREST_VERSION=1.3
-ARG JUNIT_VERSION=4.13.2
-ARG GRPC_VERSION=1_48_1
-ARG BEAM_VENDOR_GRPC_VERSION=0.1
-ARG KAFKA_CLIENTS_VERSION=2.3.1
+FROM maven:3.8.6-openjdk-8 as dep
+ARG BEAM_VERSION
+RUN mvn archetype:generate \
+ -DinteractiveMode=false \
+ -DarchetypeGroupId=org.apache.beam \
+ -DarchetypeArtifactId=beam-sdks-java-maven-archetypes-gcp-bom-examples \
+ -DarchetypeVersion=$BEAM_VERSION \
+ -DtargetPlatform=8 \
+ -DartifactId=pipeline-dependencies \
+ -DgroupId=org.apache.beam.samples
+WORKDIR /pipeline-dependencies/
+RUN mvn dependency:resolve && mvn -o dependency:list
+RUN mvn dependency:copy-dependencies
+
+FROM apache/beam_java8_sdk:$BEAM_VERSION
+
+ARG BEAM_VERSION
ARG SPRING_VERSION=5.3.18
+ARG KAFKA_CLIENTS_VERSION=2.3.1
ENV SERVER_IP=0.0.0.0
ENV SERVER_PORT=8080
ENV APP_WORK_DIR=/opt/playground/backend/
@@ -57,37 +66,17 @@ COPY --from=build /go/bin/server_java_backend /opt/playground/backend/
COPY --from=build /go/src/playground/backend/configs /opt/playground/backend/configs/
COPY --from=build /go/src/playground/backend/logging.properties /opt/playground/backend/
COPY --from=build /go/src/playground/backend/datasets /opt/playground/backend/datasets/
+COPY --from=dep /pipeline-dependencies/target/dependency/ /opt/apache/beam/jars/
-# Install Beam DirectRunner
-RUN wget https://repo1.maven.org/maven2/org/apache/beam/beam-runners-direct-java/$BEAM_VERSION/beam-runners-direct-java-$BEAM_VERSION.jar &&\
- mv beam-runners-direct-java-$BEAM_VERSION.jar /opt/apache/beam/jars/beam-runners-direct.jar
-# Install Beam SDK Core
-RUN wget https://repo1.maven.org/maven2/org/apache/beam/beam-sdks-java-core/$BEAM_VERSION/beam-sdks-java-core-$BEAM_VERSION.jar &&\
- mv beam-sdks-java-core-$BEAM_VERSION.jar /opt/apache/beam/jars/beam-sdks-java-core.jar
# Install Beam Examples
RUN wget https://repo1.maven.org/maven2/org/apache/beam/beam-examples-java/$BEAM_VERSION/beam-examples-java-$BEAM_VERSION.jar &&\
mv beam-examples-java-$BEAM_VERSION.jar /opt/apache/beam/jars/beam-examples-java.jar
# Install jars for Playground graphs
RUN wget https://repo1.maven.org/maven2/org/apache/beam/beam-runners-core-construction-java/$BEAM_VERSION/beam-runners-core-construction-java-$BEAM_VERSION.jar &&\
mv beam-runners-core-construction-java-$BEAM_VERSION.jar /opt/apache/beam/jars/beam-runners-core-construction-java-$BEAM_VERSION.jar
-RUN wget https://repo1.maven.org/maven2/org/apache/beam/beam-model-pipeline/$BEAM_VERSION/beam-model-pipeline-$BEAM_VERSION.jar &&\
- mv beam-model-pipeline-$BEAM_VERSION.jar /opt/apache/beam/jars/beam-model-pipeline-$BEAM_VERSION.jar
-# Install Beam SDK Core tests
RUN wget https://repo1.maven.org/maven2/org/apache/beam/beam-sdks-java-core/$BEAM_VERSION/beam-sdks-java-core-$BEAM_VERSION-tests.jar &&\
mv beam-sdks-java-core-$BEAM_VERSION-tests.jar /opt/apache/beam/jars/beam-sdks-java-core-tests.jar
-# Install Hamcrest
-RUN wget https://repo1.maven.org/maven2/org/hamcrest/hamcrest-all/$HAMCREST_VERSION/hamcrest-all-$HAMCREST_VERSION.jar &&\
- mv hamcrest-all-$HAMCREST_VERSION.jar /opt/apache/beam/jars/hamcrest-all.jar
-# Install JUnit
-RUN wget https://repo1.maven.org/maven2/junit/junit/$JUNIT_VERSION/junit-$JUNIT_VERSION.jar &&\
- mv junit-$JUNIT_VERSION.jar /opt/apache/beam/jars/junit.jar
-# Install Beam Vendor gRPC
-RUN wget https://repo1.maven.org/maven2/org/apache/beam/beam-vendor-grpc-$GRPC_VERSION/$BEAM_VENDOR_GRPC_VERSION/beam-vendor-grpc-$GRPC_VERSION-$BEAM_VENDOR_GRPC_VERSION.jar &&\
- mv beam-vendor-grpc-$GRPC_VERSION-$BEAM_VENDOR_GRPC_VERSION.jar /opt/apache/beam/jars/beam-vendor-grpc.jar
-# Install Kafka IO
-RUN wget https://repo1.maven.org/maven2/org/apache/beam/beam-sdks-java-io-kafka/$BEAM_VERSION/beam-sdks-java-io-kafka-$BEAM_VERSION.jar &&\
- mv beam-sdks-java-io-kafka-$BEAM_VERSION.jar /opt/apache/beam/jars/beam-sdks-java-io-kafka.jar
-# Install Apache Kafka Clients
+# Downgrade Apache Kafka Client to support mock kafka cluster
RUN wget https://repo1.maven.org/maven2/org/apache/kafka/kafka-clients/$KAFKA_CLIENTS_VERSION/kafka-clients-$KAFKA_CLIENTS_VERSION.jar &&\
mv kafka-clients-$KAFKA_CLIENTS_VERSION.jar /opt/apache/beam/jars/kafka-clients.jar
# Install Spring Expression
diff --git a/playground/backend/containers/java/build.gradle b/playground/backend/containers/java/build.gradle
index 9d1e22c56f0..3218bf17bc1 100644
--- a/playground/backend/containers/java/build.gradle
+++ b/playground/backend/containers/java/build.gradle
@@ -22,7 +22,7 @@ apply plugin: 'base'
applyDockerNature()
def playgroundJobServerProject = "${project.path.replace('-container', '')}"
-
+def default_beam_version = "2.44.0"
description = project(playgroundJobServerProject).description + " :: Container"
configurations {
@@ -69,9 +69,9 @@ docker {
project.docker_image_default_repo_root)
files "./build/"
tags containerImageTags()
- buildArgs(['BASE_IMAGE': project.rootProject.hasProperty(["base-image"]) ?
- project.rootProject["base-image"] :
- "apache/beam_java8_sdk" ])
+ buildArgs(['BEAM_VERSION': project.rootProject.hasProperty(["sdk-tag"]) ?
+ project.rootProject["sdk-tag"] :
+ default_beam_version ])
}
// Ensure that we build the required resources and copy and file dependencies from related projects