You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by yh...@apache.org on 2026/04/28 14:14:34 UTC
(beam) branch master updated: Fix LineagePluginTest on runners other than Direct runner (#38296)
This is an automated email from the ASF dual-hosted git repository.
Abacn pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new e51fb74dbbf Fix LineagePluginTest on runners other than Direct runner (#38296)
e51fb74dbbf is described below
commit e51fb74dbbf55f2c4ad5e03adffa042f063b68a2
Author: Yi Hu <ya...@google.com>
AuthorDate: Tue Apr 28 10:14:26 2026 -0400
Fix LineagePluginTest on runners other than Direct runner (#38296)
---
...beam_PostCommit_Java_ValidatesRunner_Samza.json | 3 +-
.../apache/beam/sdk/lineage/LineagePluginTest.java | 58 ++++++----------------
2 files changed, 18 insertions(+), 43 deletions(-)
diff --git a/.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Samza.json b/.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Samza.json
index db03186ab40..6acc30e8280 100644
--- a/.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Samza.json
+++ b/.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Samza.json
@@ -4,5 +4,6 @@
"comment": "Modify this file in a trivial way to cause this test suite to run",
"https://github.com/apache/beam/pull/31156": "noting that PR #31156 should run this test",
"https://github.com/apache/beam/pull/31270": "re-add specialized Samza translation of Redistribute",
- "https://github.com/apache/beam/pull/35159": "moving WindowedValue and making an interface"
+ "https://github.com/apache/beam/pull/35159": "moving WindowedValue and making an interface",
+ "modification": 1
}
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/lineage/LineagePluginTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/lineage/LineagePluginTest.java
index ff50dc1ffbe..fd78dd200bf 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/lineage/LineagePluginTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/lineage/LineagePluginTest.java
@@ -26,7 +26,6 @@ import static org.junit.Assert.assertEquals;
import java.util.Arrays;
import java.util.List;
-import org.apache.beam.sdk.PipelineResult;
import org.apache.beam.sdk.metrics.Lineage;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
import org.apache.beam.sdk.testing.NeedsRunner;
@@ -52,6 +51,8 @@ public class LineagePluginTest {
private static final Logger LOG = LoggerFactory.getLogger(LineagePluginTest.class);
+ @Rule public TestPipeline testPipeline = createTestPipelineWithLineage();
+
/**
* TestWatcher that logs detailed lineage diagnostics only when tests fail. This keeps successful
* test output clean while providing deep debugging for failures.
@@ -89,13 +90,11 @@ public class LineagePluginTest {
}
/** Helper to create a TestPipeline with test lineage configured. */
- private TestPipeline createTestPipelineWithLineage() {
- LineageOptions options = PipelineOptionsFactory.create().as(LineageOptions.class);
+ private static TestPipeline createTestPipelineWithLineage() {
+ TestPipeline testPipeline = TestPipeline.create();
+ LineageOptions options = testPipeline.getOptions().as(LineageOptions.class);
options.setLineageType(TestLineage.class);
- TestPipeline pipeline = TestPipeline.fromOptions(options);
- // Disable enforcement since we're not using @Rule
- pipeline.enableAbandonedNodeEnforcement(false);
- return pipeline;
+ return testPipeline;
}
@Test
@@ -120,16 +119,11 @@ public class LineagePluginTest {
@Test
@Category(NeedsRunner.class)
public void testLineageIntegrationWithSimpleFQN() {
- // Create pipeline with test lineage enabled - Lineage will be initialized during pipeline.run()
- TestPipeline pipeline = createTestPipelineWithLineage();
-
// Run pipeline that records lineage
- pipeline
+ testPipeline
.apply(Create.of("a", "b", "c"))
.apply(ParDo.of(new RecordSourceLineageDoFn("testsystem", Arrays.asList("db", "table"))));
-
- PipelineResult result = pipeline.run();
- result.waitUntilFinish();
+ testPipeline.run();
// Verify lineage was recorded
List<String> sources = TestLineage.getRecordedSources();
@@ -139,11 +133,8 @@ public class LineagePluginTest {
@Test
@Category(NeedsRunner.class)
public void testLineageIntegrationWithSubtype() {
- // Create pipeline with test lineage enabled - Lineage will be initialized during pipeline.run()
- TestPipeline pipeline = createTestPipelineWithLineage();
-
// Run pipeline that records lineage with subtype
- pipeline
+ testPipeline
.apply(Create.of(1, 2, 3))
.apply(
ParDo.of(
@@ -151,9 +142,7 @@ public class LineagePluginTest {
"spanner",
"table",
Arrays.asList("project", "instance", "database", "table"))));
-
- PipelineResult result = pipeline.run();
- result.waitUntilFinish();
+ testPipeline.run();
// Verify lineage was recorded with subtype
List<String> sources = TestLineage.getRecordedSources();
@@ -163,19 +152,14 @@ public class LineagePluginTest {
@Test
@Category(NeedsRunner.class)
public void testLineageIntegrationWithLastSegmentSeparator() {
- // Create pipeline with test lineage enabled - Lineage will be initialized during pipeline.run()
- TestPipeline pipeline = createTestPipelineWithLineage();
-
// Run pipeline that records lineage with custom separator
- pipeline
+ testPipeline
.apply(Create.of("x", "y", "z"))
.apply(
ParDo.of(
new RecordSourceLineageWithSeparatorDoFn(
"gcs", Arrays.asList("bucket", "path/to/file.txt"), "/")));
-
- PipelineResult result = pipeline.run();
- result.waitUntilFinish();
+ testPipeline.run();
// Verify lineage was recorded with separator
List<String> sources = TestLineage.getRecordedSources();
@@ -185,16 +169,11 @@ public class LineagePluginTest {
@Test
@Category(NeedsRunner.class)
public void testLineageIntegrationWithBothSourcesAndSinks() {
- // Create pipeline with test lineage enabled - Lineage will be initialized during pipeline.run()
- TestPipeline pipeline = createTestPipelineWithLineage();
-
// Run pipeline that records both source and sink lineage
- pipeline
+ testPipeline
.apply(Create.of("data1", "data2"))
.apply(ParDo.of(new RecordBothSourceAndSinkLineageDoFn()));
-
- PipelineResult result = pipeline.run();
- result.waitUntilFinish();
+ testPipeline.run();
// Verify both source and sink lineage were recorded
List<String> sources = TestLineage.getRecordedSources();
@@ -207,16 +186,11 @@ public class LineagePluginTest {
@Test
@Category(NeedsRunner.class)
public void testLineageIntegrationWithMultipleElements() {
- // Create pipeline with test lineage enabled - Lineage will be initialized during pipeline.run()
- TestPipeline pipeline = createTestPipelineWithLineage();
-
// Run pipeline with multiple elements to test thread safety
- pipeline
+ testPipeline
.apply(Create.of(1, 2, 3, 4, 5, 6, 7, 8, 9, 10))
.apply(ParDo.of(new RecordSourceLineageDoFn("system", Arrays.asList("resource"))));
-
- PipelineResult result = pipeline.run();
- result.waitUntilFinish();
+ testPipeline.run();
// Verify lineage was recorded for all elements (may have duplicates)
List<String> sources = TestLineage.getRecordedSources();