You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by yh...@apache.org on 2025/07/16 14:02:45 UTC
(beam) 01/01: Fix Java8 tests to have gradle run on Java11
This is an automated email from the ASF dual-hosted git repository.
yhu pushed a commit to branch fixjava8test
in repository https://gitbox.apache.org/repos/asf/beam.git
commit ad82fceaa1e7be9cdc97a89fe92f0018da8d58bf
Author: Yi Hu <ya...@google.com>
AuthorDate: Wed Jul 16 10:02:05 2025 -0400
Fix Java8 tests to have gradle run on Java11
---
...it_Java_ValidatesRunner_Direct_JavaVersions.yml | 11 ++---------
...PostCommit_Java_ValidatesRunner_Flink_Java8.yml | 14 ++------------
...PostCommit_Java_ValidatesRunner_Spark_Java8.yml | 14 ++------------
sdks/go/test/build.gradle | 16 ++++++++--------
sdks/go/test/run_validatesrunner_tests.sh | 22 ++++------------------
5 files changed, 18 insertions(+), 59 deletions(-)
diff --git a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Direct_JavaVersions.yml b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Direct_JavaVersions.yml
index 11b18548249..d0499b65c5a 100644
--- a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Direct_JavaVersions.yml
+++ b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Direct_JavaVersions.yml
@@ -80,20 +80,13 @@ jobs:
java-version: |
${{ matrix.java_version }}
11
- - name: run jar Java${{ matrix.java_version }} script
- run: |
- ./gradlew :runners:direct-java:shadowJar :runners:direct-java:shadowTestJar \
- -Dorg.gradle.java.home=$JAVA_HOME_${{ matrix.java_version }}_X64
- name: run validatesRunner Java${{ matrix.java_version }} script
uses: ./.github/actions/gradle-command-self-hosted-action
with:
gradle-command: :runners:direct-java:validatesRunner
arguments: |
- -x shadowJar \
- -x shadowTestJar \
- -x compileJava \
- -x compileTestJava \
- -Dorg.gradle.java.home=$JAVA_HOME_${{ matrix.java_version }}_X64 \
+ -PtestJavaVersion=${{ matrix.java_version }}
+ -Pjava${{ matrix.java_version }}Home=$JAVA_HOME_${{ matrix.java_version }}_X64 \
- name: Archive JUnit Test Results
uses: actions/upload-artifact@v4
if: ${{ !success() }}
diff --git a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Flink_Java8.yml b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Flink_Java8.yml
index 7f7b9c9270d..9b061028cbc 100644
--- a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Flink_Java8.yml
+++ b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Flink_Java8.yml
@@ -78,23 +78,13 @@ jobs:
java-version: |
8
11
- - name: run jar Java8 script
- run: |
- ./gradlew :runners:flink:1.19:jar :runners:flink:1.19:testJar
- name: run validatesRunner Java8 script
uses: ./.github/actions/gradle-command-self-hosted-action
with:
gradle-command: :runners:flink:1.19:validatesRunner
arguments: |
- -x shadowJar \
- -x shadowTestJar \
- -x compileJava \
- -x compileTestJava \
- -x jar \
- -x testJar \
- -x classes \
- -x testClasses \
- -Dorg.gradle.java.home=$JAVA_HOME_8_X64 \
+ -PtestJavaVersion=8 \
+ -Pjava8Home=$JAVA_HOME_8_X64 \
max-workers: 12
- name: Archive JUnit Test Results
uses: actions/upload-artifact@v4
diff --git a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Spark_Java8.yml b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Spark_Java8.yml
index 0852fcdf4af..dae408e4346 100644
--- a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Spark_Java8.yml
+++ b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Spark_Java8.yml
@@ -78,23 +78,13 @@ jobs:
java-version: |
8
11
- - name: run jar Java8 script
- run: |
- ./gradlew :runners:spark:3:jar :runners:spark:3:testJar
- name: run validatesRunner Java8 script
uses: ./.github/actions/gradle-command-self-hosted-action
with:
gradle-command: :runners:spark:3:validatesRunner
arguments: |
- -x shadowJar \
- -x shadowTestJar \
- -x compileJava \
- -x compileTestJava \
- -x jar \
- -x testJar \
- -x classes \
- -x testClasses \
- -Dorg.gradle.java.home=$JAVA_HOME_8_X64 \
+ -PtestJavaVersion=8 \
+ -Pjava8Home=$JAVA_HOME_8_X64 \
max-workers: 12
- name: Archive JUnit Test Results
uses: actions/upload-artifact@v4
diff --git a/sdks/go/test/build.gradle b/sdks/go/test/build.gradle
index 74b6a10cad4..5576c40c0aa 100644
--- a/sdks/go/test/build.gradle
+++ b/sdks/go/test/build.gradle
@@ -41,7 +41,7 @@ task dataflowValidatesRunner() {
]
exec {
if (fork_java_home) {
- environment "JAVA_HOME", fork_java_home
+ environment "JAVA_HOME_JOB_SERVER", fork_java_home
}
executable "sh"
args "-c", "./run_validatesrunner_tests.sh ${options.join(' ')}"
@@ -66,7 +66,7 @@ task dataflowValidatesRunnerARM64() {
]
exec {
if (fork_java_home) {
- environment "JAVA_HOME", fork_java_home
+ environment "JAVA_HOME_JOB_SERVER", fork_java_home
}
executable "sh"
args "-c", "./run_validatesrunner_tests.sh ${options.join(' ')}"
@@ -95,7 +95,7 @@ task flinkValidatesRunner {
]
exec {
if (fork_java_home) {
- environment "JAVA_HOME", fork_java_home
+ environment "JAVA_HOME_JOB_SERVER", fork_java_home
}
executable "sh"
args "-c", "./run_validatesrunner_tests.sh ${options.join(' ')}"
@@ -123,7 +123,7 @@ task samzaValidatesRunner {
]
exec {
if (fork_java_home) {
- environment "JAVA_HOME", fork_java_home
+ environment "JAVA_HOME_JOB_SERVER", fork_java_home
}
executable "sh"
args "-c", "./run_validatesrunner_tests.sh ${options.join(' ')}"
@@ -151,7 +151,7 @@ task sparkValidatesRunner {
]
exec {
if (fork_java_home) {
- environment "JAVA_HOME", fork_java_home
+ environment "JAVA_HOME_JOB_SERVER", fork_java_home
}
executable "sh"
args "-c", "./run_validatesrunner_tests.sh ${options.join(' ')}"
@@ -190,7 +190,7 @@ tasks.register("ulrValidatesRunner") {
}
exec {
if (fork_java_home) {
- environment "JAVA_HOME", fork_java_home
+ environment "JAVA_HOME_JOB_SERVER", fork_java_home
}
executable "sh"
args "-c", ". ${envdir}/bin/activate && ./run_validatesrunner_tests.sh ${options.join(' ')}"
@@ -217,7 +217,7 @@ task prismValidatesRunner {
]
exec {
if (fork_java_home) {
- environment "JAVA_HOME", fork_java_home
+ environment "JAVA_HOME_JOB_SERVER", fork_java_home
}
executable "sh"
args "-c", "./run_validatesrunner_tests.sh ${options.join(' ')}"
@@ -262,7 +262,7 @@ ext.goIoValidatesRunnerTask = { proj, name, scriptOpts, pipelineOpts ->
logger.info("Running the command: sh -c ./run_validatesrunner_tests.sh ${options.join(' ')}")
exec {
if (fork_java_home) {
- environment "JAVA_HOME", fork_java_home
+ environment "JAVA_HOME_JOB_SERVER", fork_java_home
}
executable "sh"
args "-c", "./run_validatesrunner_tests.sh ${options.join(' ')}"
diff --git a/sdks/go/test/run_validatesrunner_tests.sh b/sdks/go/test/run_validatesrunner_tests.sh
index 4278c817d0a..2e1b5fa3f39 100755
--- a/sdks/go/test/run_validatesrunner_tests.sh
+++ b/sdks/go/test/run_validatesrunner_tests.sh
@@ -259,8 +259,8 @@ s.close()
TMPDIR=$(mktemp -d)
-if [[ -n "$JAVA_HOME" ]]; then
- JAVA_CMD="$JAVA_HOME/bin/java"
+if [[ -n "$JAVA_HOME_JOB_SERVER" ]]; then
+ JAVA_CMD="$JAVA_HOME_JOB_SERVER/bin/java"
else
JAVA_CMD="java"
fi
@@ -374,18 +374,7 @@ if [[ "$RUNNER" == "dataflow" ]]; then
CONTAINER=us.gcr.io/$PROJECT/$USER/beam_go_sdk
echo "Using container $CONTAINER"
- # TODO(https://github.com/apache/beam/issues/27674): remove this branch once the jenkins VM can build multiarch, or jenkins is deprecated.
- if [[ "$USER" == "jenkins" ]]; then
- ./gradlew :sdks:go:container:docker -Pdocker-repository-root=us.gcr.io/$PROJECT/$USER -Pdocker-tag=$TAG
-
- # Verify it exists
- docker images | grep $TAG
-
- # Push the container
- gcloud docker -- push $CONTAINER:$TAG
- else
- ./gradlew :sdks:go:container:docker -Pdocker-repository-root=us.gcr.io/$PROJECT/$USER -Pdocker-tag=$TAG -Pcontainer-architecture-list=arm64,amd64 -Ppush-containers
- fi
+ ./gradlew :sdks:go:container:docker -Pdocker-repository-root=us.gcr.io/$PROJECT/$USER -Pdocker-tag=$TAG -Pcontainer-architecture-list=arm64,amd64 -Ppush-containers
if [[ -n "$TEST_EXPANSION_ADDR" || -n "$IO_EXPANSION_ADDR" || -n "$SCHEMAIO_EXPANSION_ADDR" || -n "$DEBEZIUMIO_EXPANSION_ADDR" ]]; then
ARGS="$ARGS --experiments=use_portable_job_submission"
@@ -395,7 +384,7 @@ if [[ "$RUNNER" == "dataflow" ]]; then
JAVA_TAG=$(date +%Y%m%d-%H%M%S)
JAVA_CONTAINER=us.gcr.io/$PROJECT/$USER/beam_java11_sdk
echo "Using container $JAVA_CONTAINER for cross-language java transforms"
- ./gradlew :sdks:java:container:java11:docker -Pdocker-repository-root=us.gcr.io/$PROJECT/$USER -Pdocker-tag=$JAVA_TAG -Pjava11Home=$JAVA11_HOME
+ ./gradlew :sdks:java:container:java11:docker -Pdocker-repository-root=us.gcr.io/$PROJECT/$USER -Pdocker-tag=$JAVA_TAG
# Verify it exists
docker images | grep $JAVA_TAG
@@ -457,9 +446,6 @@ if [[ "$RUNNER" == "dataflow" ]]; then
# Note: we don't delete the multi-arch containers here because this command only deletes the manifest list with the tag,
# the associated container images can't be deleted because they are not tagged. However, multi-arch containers that are
# older than 6 weeks old are deleted by stale_dataflow_prebuilt_image_cleaner.sh that runs daily.
- if [[ "$USER" == "jenkins" ]]; then
- gcloud --quiet container images delete $CONTAINER:$TAG || echo "Failed to delete container"
- fi
if [[ -n "$TEST_EXPANSION_ADDR" || -n "$IO_EXPANSION_ADDR" || -n "$SCHEMAIO_EXPANSION_ADDR" || -n "$DEBEZIUMIO_EXPANSION_ADDR" ]]; then
# Delete the java cross-language container locally and remotely
docker rmi $JAVA_CONTAINER:$JAVA_TAG || echo "Failed to remove container"