You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by yh...@apache.org on 2024/04/01 14:29:15 UTC
(beam) branch master updated: add get beam latest release and fix GCP auth (#30751)
This is an automated email from the ASF dual-hosted git repository.
yhu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new fecc9d995e4 add get beam latest release and fix GCP auth (#30751)
fecc9d995e4 is described below
commit fecc9d995e41cf7f4ffc7963a883a2e6444740f4
Author: Vlado Djerek <dj...@gmail.com>
AuthorDate: Mon Apr 1 16:29:08 2024 +0200
add get beam latest release and fix GCP auth (#30751)
* add get beam latest release and fix GCP auth
* added param to change max worker on dataflow job. Disabled shedule until next release
* fix spotless issues
---
.../beam_PostCommit_Java_IO_Performance_Tests.yml | 27 ++++++++++++++--------
.../beam/it/gcp/bigquery/BigQueryStreamingLT.java | 6 +++++
.../apache/beam/it/gcp/bigtable/BigTableIOLT.java | 6 +++++
.../apache/beam/it/gcp/storage/FileBasedIOLT.java | 6 +++++
4 files changed, 36 insertions(+), 9 deletions(-)
diff --git a/.github/workflows/beam_PostCommit_Java_IO_Performance_Tests.yml b/.github/workflows/beam_PostCommit_Java_IO_Performance_Tests.yml
index 588da4ea61a..63319789cb0 100644
--- a/.github/workflows/beam_PostCommit_Java_IO_Performance_Tests.yml
+++ b/.github/workflows/beam_PostCommit_Java_IO_Performance_Tests.yml
@@ -18,8 +18,8 @@ name: PostCommit Java IO Performance Tests
on:
issue_comment:
types: [created]
- schedule:
- - cron: '0 4/6 * * *'
+# schedule:
+# - cron: '0 4/6 * * *'
pull_request_target:
paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Java_IO_Performance_Tests.json']
workflow_dispatch:
@@ -73,28 +73,37 @@ jobs:
comment_phrase: ${{ matrix.job_phrase }} ${{ matrix.test_case }}
github_token: ${{ secrets.GITHUB_TOKEN }}
github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.test_case }})
+ - name: Get Beam latest release
+ if: github.event_name == 'schedule' #This has scheduled runs run against the latest release
+ run: |
+ BEAM_VERSION=$(curl -s https://api.github.com/repos/apache/beam/releases/latest | jq -r '.tag_name')
+ echo "BEAM_VERSION=${BEAM_VERSION}" >> $GITHUB_ENV
- name: Checkout release branch
if: github.event_name == 'schedule' #This has scheduled runs run against the latest release
uses: actions/checkout@v4
with:
- ref: v2.50.0 #TODO(https://github.com/apache/beam/issues/28330) automate updating this
+ ref: ${{ env.BEAM_VERSION }}
repository: apache/beam
- name: Setup environment
uses: ./.github/actions/setup-environment-action
with:
java-version: 8
- name: Authenticate on GCP
- uses: google-github-actions/setup-gcloud@v0
+ uses: google-github-actions/auth@v2
+ with:
+ credentials_json: ${{ secrets.GCP_SA_KEY }}
+ project_id: ${{ secrets.GCP_PROJECT_ID }}
+ token_format: 'access_token'
+ - name: Setup gcloud
+ uses: google-github-actions/setup-gcloud@v2
with:
- service_account_email: ${{ secrets.GCP_SA_EMAIL }}
- service_account_key: ${{ secrets.GCP_SA_KEY }}
project_id: ${{ secrets.GCP_PROJECT_ID }}
- export_default_credentials: true
+ skip_install: true
- name: run scheduled javaPostcommitIOPerformanceTests script
if: github.event_name == 'schedule' #This ensures only scheduled runs publish metrics publicly by changing which exportTable is configured
uses: ./.github/actions/gradle-command-self-hosted-action
with:
- gradle-command: :it:${{ matrix.test_case }}
+ gradle-command: :it:${{ matrix.test_case }} -DmaxNumWorkers=10
env:
exportDataset: performance_tests
exportTable: io_performance_metrics
@@ -102,7 +111,7 @@ jobs:
if: github.event_name != 'schedule'
uses: ./.github/actions/gradle-command-self-hosted-action
with:
- gradle-command: :it:${{ matrix.test_case }}
+ gradle-command: :it:${{ matrix.test_case }} -DmaxNumWorkers=10
env:
exportDataset: performance_tests
exportTable: io_performance_metrics_test
diff --git a/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/bigquery/BigQueryStreamingLT.java b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/bigquery/BigQueryStreamingLT.java
index 214d2745f03..28a4f399e8e 100644
--- a/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/bigquery/BigQueryStreamingLT.java
+++ b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/bigquery/BigQueryStreamingLT.java
@@ -281,6 +281,9 @@ public class BigQueryStreamingLT extends IOLoadTestBase {
.setSdk(PipelineLauncher.Sdk.JAVA)
.setPipeline(fileLoadsPipeline)
.addParameter("runner", config.getRunner())
+ .addParameter(
+ "maxNumWorkers",
+ TestProperties.getProperty("maxNumWorkers", "10", TestProperties.Type.PROPERTY))
.build();
// Don't use PipelineOperator because we don't want to wait on this batch job
@@ -360,6 +363,9 @@ public class BigQueryStreamingLT extends IOLoadTestBase {
.addParameter("runner", config.getRunner())
.addParameter("streaming", "true")
.addParameter("experiments", GcpOptions.STREAMING_ENGINE_EXPERIMENT)
+ .addParameter(
+ "maxNumWorkers",
+ TestProperties.getProperty("maxNumWorkers", "10", TestProperties.Type.PROPERTY))
.build();
// Launch job
PipelineLauncher.LaunchInfo storageApiInfo =
diff --git a/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/bigtable/BigTableIOLT.java b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/bigtable/BigTableIOLT.java
index a6516863b8d..9770a40cb83 100644
--- a/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/bigtable/BigTableIOLT.java
+++ b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/bigtable/BigTableIOLT.java
@@ -178,6 +178,9 @@ public class BigTableIOLT extends IOLoadTestBase {
.setSdk(PipelineLauncher.Sdk.JAVA)
.setPipeline(writePipeline)
.addParameter("runner", configuration.getRunner())
+ .addParameter(
+ "maxNumWorkers",
+ TestProperties.getProperty("maxNumWorkers", "10", TestProperties.Type.PROPERTY))
.build();
return pipelineLauncher.launch(project, region, options);
@@ -200,6 +203,9 @@ public class BigTableIOLT extends IOLoadTestBase {
.setSdk(PipelineLauncher.Sdk.JAVA)
.setPipeline(readPipeline)
.addParameter("runner", configuration.getRunner())
+ .addParameter(
+ "maxNumWorkers",
+ TestProperties.getProperty("maxNumWorkers", "10", TestProperties.Type.PROPERTY))
.build();
return pipelineLauncher.launch(project, region, options);
diff --git a/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/storage/FileBasedIOLT.java b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/storage/FileBasedIOLT.java
index a36f3b340e8..82caea31218 100644
--- a/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/storage/FileBasedIOLT.java
+++ b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/storage/FileBasedIOLT.java
@@ -184,6 +184,9 @@ public class FileBasedIOLT extends IOLoadTestBase {
.setSdk(PipelineLauncher.Sdk.JAVA)
.setPipeline(writePipeline)
.addParameter("runner", configuration.runner)
+ .addParameter(
+ "maxNumWorkers",
+ TestProperties.getProperty("maxNumWorkers", "10", TestProperties.Type.PROPERTY))
.build();
PipelineLauncher.LaunchInfo writeInfo = pipelineLauncher.launch(project, region, writeOptions);
PipelineOperator.Result writeResult =
@@ -198,6 +201,9 @@ public class FileBasedIOLT extends IOLoadTestBase {
.setSdk(PipelineLauncher.Sdk.JAVA)
.setPipeline(readPipeline)
.addParameter("runner", configuration.runner)
+ .addParameter(
+ "maxNumWorkers",
+ TestProperties.getProperty("maxNumWorkers", "10", TestProperties.Type.PROPERTY))
.build();
PipelineLauncher.LaunchInfo readInfo = pipelineLauncher.launch(project, region, readOptions);
PipelineOperator.Result readResult =