You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by yh...@apache.org on 2025/10/09 21:26:29 UTC

(beam) branch master updated: Fix XVR JavaUsingPython tests using dev Beam at expansion (#36444)

This is an automated email from the ASF dual-hosted git repository.

yhu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/master by this push:
     new 08c96f2c6e2 Fix XVR JavaUsingPython tests using dev Beam at expansion (#36444)
08c96f2c6e2 is described below

commit 08c96f2c6e257dfec898414612c51ec698d0c838
Author: Yi Hu <ya...@google.com>
AuthorDate: Thu Oct 9 17:26:22 2025 -0400

    Fix XVR JavaUsingPython tests using dev Beam at expansion (#36444)
    
    * Uses test pipeline
    
    * Set custom beam requirement
    
    * Fix spotless
---
 runners/flink/job-server/flink_job_server.gradle          |  1 +
 runners/google-cloud-dataflow-java/build.gradle           |  1 +
 runners/samza/job-server/build.gradle                     |  1 +
 runners/spark/job-server/spark_job_server.gradle          |  1 +
 .../extensions/python/PythonExternalTransformTest.java    | 15 +++++++++------
 sdks/python/test-suites/direct/xlang/build.gradle         |  1 +
 6 files changed, 14 insertions(+), 6 deletions(-)

diff --git a/runners/flink/job-server/flink_job_server.gradle b/runners/flink/job-server/flink_job_server.gradle
index 90890a7d585..d8a818ff84c 100644
--- a/runners/flink/job-server/flink_job_server.gradle
+++ b/runners/flink/job-server/flink_job_server.gradle
@@ -269,6 +269,7 @@ createCrossLanguageValidatesRunnerTask(
     "--environmentCacheMillis=10000",
     "--experiments=beam_fn_api",
     "--parallelism=2",
+    "--customBeamRequirement=${project.project(":sdks:python").projectDir}/build/apache-beam.tar.gz",
   ],
   goScriptOptions: [
     "--runner flink",
diff --git a/runners/google-cloud-dataflow-java/build.gradle b/runners/google-cloud-dataflow-java/build.gradle
index 85f8b583c34..8729bc2032c 100644
--- a/runners/google-cloud-dataflow-java/build.gradle
+++ b/runners/google-cloud-dataflow-java/build.gradle
@@ -486,6 +486,7 @@ createCrossLanguageValidatesRunnerTask(
     "--tempRoot=${dataflowValidatesTempRoot}",
     "--sdkContainerImage=${dockerJavaImageContainer}:${dockerTag}",
     "--sdkHarnessContainerImageOverrides=.*python.*,${dockerPythonImageContainer}:${dockerTag}",
+    "--customBeamRequirement=${project.project(":sdks:python").projectDir}/build/apache-beam.tar.gz",
   ],
   pytestOptions: [
     "--capture=no",
diff --git a/runners/samza/job-server/build.gradle b/runners/samza/job-server/build.gradle
index 05f6de39254..7ffb2becd6d 100644
--- a/runners/samza/job-server/build.gradle
+++ b/runners/samza/job-server/build.gradle
@@ -243,6 +243,7 @@ createCrossLanguageValidatesRunnerTask(
                 "--jobEndpoint=localhost:${jobPort}",
                 "--environmentCacheMillis=10000",
                 "--experiments=beam_fn_api",
+                "--customBeamRequirement=${project.project(":sdks:python").projectDir}/build/apache-beam.tar.gz",
         ],
         goScriptOptions: [
             "--runner samza",
diff --git a/runners/spark/job-server/spark_job_server.gradle b/runners/spark/job-server/spark_job_server.gradle
index 90109598ed6..7e2deaf6e39 100644
--- a/runners/spark/job-server/spark_job_server.gradle
+++ b/runners/spark/job-server/spark_job_server.gradle
@@ -294,6 +294,7 @@ createCrossLanguageValidatesRunnerTask(
     "--jobEndpoint=localhost:${jobPort}",
     "--environmentCacheMillis=10000",
     "--experiments=beam_fn_api",
+    "--customBeamRequirement=${project.project(":sdks:python").projectDir}/build/apache-beam.tar.gz",
   ],
   goScriptOptions: [
     "--runner spark",
diff --git a/sdks/java/extensions/python/src/test/java/org/apache/beam/sdk/extensions/python/PythonExternalTransformTest.java b/sdks/java/extensions/python/src/test/java/org/apache/beam/sdk/extensions/python/PythonExternalTransformTest.java
index a1e1dade513..f522a4c409f 100644
--- a/sdks/java/extensions/python/src/test/java/org/apache/beam/sdk/extensions/python/PythonExternalTransformTest.java
+++ b/sdks/java/extensions/python/src/test/java/org/apache/beam/sdk/extensions/python/PythonExternalTransformTest.java
@@ -33,6 +33,7 @@ import org.apache.beam.sdk.schemas.Schema;
 import org.apache.beam.sdk.schemas.SchemaTranslation;
 import org.apache.beam.sdk.schemas.logicaltypes.MicrosInstant;
 import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.testing.TestPipeline;
 import org.apache.beam.sdk.testing.UsesPythonExpansionService;
 import org.apache.beam.sdk.testing.ValidatesRunner;
 import org.apache.beam.sdk.transforms.Create;
@@ -43,6 +44,7 @@ import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.Row;
 import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList;
 import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap;
+import org.junit.Rule;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 import org.junit.runner.RunWith;
@@ -50,28 +52,29 @@ import org.junit.runners.JUnit4;
 
 @RunWith(JUnit4.class)
 public class PythonExternalTransformTest implements Serializable {
+  @Rule public transient TestPipeline testPipeline = TestPipeline.create();
 
   @Test
   @Category({ValidatesRunner.class, UsesPythonExpansionService.class})
   public void trivialPythonTransform() {
-    Pipeline p = Pipeline.create();
     PCollection<String> output =
-        p.apply(Create.of(KV.of("A", "x"), KV.of("A", "y"), KV.of("B", "z")))
+        testPipeline
+            .apply(Create.of(KV.of("A", "x"), KV.of("A", "y"), KV.of("B", "z")))
             .apply(
                 PythonExternalTransform
                     .<PCollection<KV<String, String>>, PCollection<KV<String, Iterable<String>>>>
                         from("apache_beam.GroupByKey"))
             .apply(Keys.create());
     PAssert.that(output).containsInAnyOrder("A", "B");
-    // TODO: Run this on a multi-language supporting runner.
+    testPipeline.run();
   }
 
   @Test
   @Category({ValidatesRunner.class, UsesPythonExpansionService.class})
   public void pythonTransformWithDependencies() {
-    Pipeline p = Pipeline.create();
     PCollection<String> output =
-        p.apply(Create.of("elephant", "mouse", "sheep"))
+        testPipeline
+            .apply(Create.of("elephant", "mouse", "sheep"))
             .apply(
                 PythonExternalTransform.<PCollection<String>, PCollection<String>>from(
                         "apache_beam.Map")
@@ -79,7 +82,7 @@ public class PythonExternalTransformTest implements Serializable {
                     .withExtraPackages(ImmutableList.of("inflection"))
                     .withOutputCoder(StringUtf8Coder.of()));
     PAssert.that(output).containsInAnyOrder("elephants", "mice", "sheep");
-    // TODO: Run this on a multi-language supporting runner.
+    testPipeline.run();
   }
 
   @Test
diff --git a/sdks/python/test-suites/direct/xlang/build.gradle b/sdks/python/test-suites/direct/xlang/build.gradle
index 3003329aef5..602b633e350 100644
--- a/sdks/python/test-suites/direct/xlang/build.gradle
+++ b/sdks/python/test-suites/direct/xlang/build.gradle
@@ -62,6 +62,7 @@ createCrossLanguageValidatesRunnerTask(
     "--jobEndpoint=localhost:${jobPort}",
     "--environmentCacheMillis=10000",
     "--experiments=beam_fn_api",
+    "--customBeamRequirement=${project.project(":sdks:python").projectDir}/build/apache-beam.tar.gz",
   ],
   goScriptOptions: [
     "--runner portable",