(beam) branch fix-postrelease-test updated: Support passing experiments to archetype pipeline

[yh...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

yhu pushed a commit to branch fix-postrelease-test
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/fix-postrelease-test by this push:
     new 9d47a47ca15 Support passing experiments to archetype pipeline
9d47a47ca15 is described below

commit 9d47a47ca15e2851d7d975e1a484c6c53a2f35f7
Author: Yi Hu <ya...@google.com>
AuthorDate: Thu Jul 17 16:42:18 2025 -0400

    Support passing experiments to archetype pipeline
---
 .../main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy |  7 ++++++-
 release/src/main/groovy/MobileGamingCommands.groovy            |  3 +++
 release/src/main/groovy/TestScripts.groovy                     | 10 ++++++++++
 release/src/main/groovy/quickstart-java-dataflow.groovy        |  3 ++-
 4 files changed, 21 insertions(+), 2 deletions(-)

diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
index 9fb6b89db76..68ab72529f9 100644
--- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
+++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
@@ -2562,7 +2562,8 @@ class BeamModulePlugin implements Plugin<Project> {
 
       def taskName = "run${config.type}Java${config.runner}"
       def releaseVersion = project.findProperty('ver') ?: project.version
-      def releaseRepo = project.findProperty('repourl') ?: 'https://repository.apache.org/content/repositories/snapshots'
+      final def snapshotUrl = 'https://repository.apache.org/content/repositories/snapshots'
+      def releaseRepo = project.findProperty('repourl') ?: snapshotUrl
       // shared maven local path for maven archetype projects
       def sharedMavenLocal = project.findProperty('mavenLocalPath') ?: ''
       def argsNeeded = [
@@ -2587,6 +2588,10 @@ class BeamModulePlugin implements Plugin<Project> {
       if (sharedMavenLocal) {
         argsNeeded.add("--mavenLocalPath=${sharedMavenLocal}")
       }
+      if (config.runner == 'Dataflow' && releaseRepo == snapshotUrl) {
+        // override Dataflow worker/SDK harness Java when run on snapshot
+        argsNeeded.add('--experiments=use_staged_dataflow_worker_jar,use_runner_v2')
+      }
       project.evaluationDependsOn(':release')
       project.task(taskName, dependsOn: ':release:classes', type: JavaExec) {
         group = "Verification"
diff --git a/release/src/main/groovy/MobileGamingCommands.groovy b/release/src/main/groovy/MobileGamingCommands.groovy
index 197cbd7a1cd..1b5bc234a27 100644
--- a/release/src/main/groovy/MobileGamingCommands.groovy
+++ b/release/src/main/groovy/MobileGamingCommands.groovy
@@ -108,6 +108,9 @@ class MobileGamingCommands {
         break
       default:
         testScripts.error("Cannot find example ${exampleName} in archetypes.")
+      if (testScripts.experiments()) {
+        args["experiments"] = testScripts.experiments()
+      }
     }
 
     StringBuilder exampleArgs = new StringBuilder("--tempLocation=gs://${testScripts.gcsBucket()}/tmp --runner=${runner} ")
diff --git a/release/src/main/groovy/TestScripts.groovy b/release/src/main/groovy/TestScripts.groovy
index 8728221bc89..84f77a7eb6f 100644
--- a/release/src/main/groovy/TestScripts.groovy
+++ b/release/src/main/groovy/TestScripts.groovy
@@ -37,6 +37,7 @@ class TestScripts {
      static String bqDataset
      static String pubsubTopic
      static String mavenLocalPath
+     static String experiments
    }
 
    def TestScripts(String[] args) {
@@ -49,6 +50,7 @@ class TestScripts {
      cli.bqDataset(args:1, "BigQuery Dataset")
      cli.pubsubTopic(args:1, "PubSub Topic")
      cli.mavenLocalPath(args:1, "Maven local path")
+     cli.experiments(args:1, "Comma seperated experiments pipeline options")
 
      def options = cli.parse(args)
      var.repoUrl = options.repourl
@@ -79,6 +81,10 @@ class TestScripts {
          var.mavenLocalPath = options.mavenLocalPath
          println "Maven local path: ${var.mavenLocalPath}"
      }
+     if (options.experiments) {
+        var.experiments = options.experiments
+        println "experiments: ${var.experiments}"
+     }
    }
 
    def ver() {
@@ -105,6 +111,10 @@ class TestScripts {
      return var.pubsubTopic
    }
 
+   def experiments() {
+     return var.experiments
+   }
+
    // Both documents the overal scenario and creates a clean temp directory
    def describe(String desc) {
      var.startDir = File.createTempDir()
diff --git a/release/src/main/groovy/quickstart-java-dataflow.groovy b/release/src/main/groovy/quickstart-java-dataflow.groovy
index fb84fedca0b..0512fde3249 100644
--- a/release/src/main/groovy/quickstart-java-dataflow.groovy
+++ b/release/src/main/groovy/quickstart-java-dataflow.groovy
@@ -33,6 +33,7 @@ t.describe 'Run Apache Beam Java SDK Quickstart - Dataflow'
     // Remove any count files
     t.run """gsutil rm gs://${t.gcsBucket()}/count* || echo 'No files'"""
 
+    def maybeExperiments = t.experiments() ? "--experiments=${ t.experiments()}" : ''
     // Run the wordcount example with the Dataflow runner
     t.run """mvn compile exec:java -q \
       -Dmaven.wagon.http.retryHandler.class=default \
@@ -43,7 +44,7 @@ t.describe 'Run Apache Beam Java SDK Quickstart - Dataflow'
       -Dexec.mainClass=org.apache.beam.examples.WordCount \
       -Dexec.args="--runner=DataflowRunner \
                    --project=${t.gcpProject()} \
-                   --region=${t.gcpRegion()} \
+                   --region=${t.gcpRegion()} ${maybeExperiments} \
                    --gcpTempLocation=gs://${t.gcsBucket()}/tmp \
                    --output=gs://${t.gcsBucket()}/counts \
                    --inputFile=gs://apache-beam-samples/shakespeare/*" \