You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by yh...@apache.org on 2024/02/21 16:10:51 UTC
(beam) branch master updated: Fix XVR test exclusions (#30354)
This is an automated email from the ASF dual-hosted git repository.
yhu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 9d2fea44b59 Fix XVR test exclusions (#30354)
9d2fea44b59 is described below
commit 9d2fea44b5982c98b626108d93275f832213b305
Author: Yi Hu <ya...@google.com>
AuthorDate: Wed Feb 21 11:10:43 2024 -0500
Fix XVR test exclusions (#30354)
* Fix XVR test exclusions
* Add trigger files
---
.../trigger_files/beam_PostCommit_Java_PVR_Spark3_Streaming.json | 0
.github/trigger_files/beam_PostCommit_Java_PVR_Spark_Batch.json | 0
.../beam_PostCommit_Java_ValidatesRunner_Dataflow_V2.json | 0
...am_PostCommit_Java_ValidatesRunner_Dataflow_V2_Streaming.json | 0
.../beam_PostCommit_Java_ValidatesRunner_Flink_Java11.json | 0
...PostCommit_Java_ValidatesRunner_SparkStructuredStreaming.json | 0
.../beam_PostCommit_Java_ValidatesRunner_Twister2.json | 0
.../trigger_files/beam_PostCommit_Java_ValidatesRunner_ULR.json | 0
.github/trigger_files/beam_PostCommit_XVR_Flink.json | 0
.../main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy | 3 +++
runners/direct-java/build.gradle | 6 ++----
runners/flink/flink_runner.gradle | 1 +
runners/flink/job-server/flink_job_server.gradle | 3 +--
runners/google-cloud-dataflow-java/build.gradle | 4 ++--
runners/jet/build.gradle | 1 +
runners/portability/java/build.gradle | 1 +
runners/samza/build.gradle | 1 +
runners/samza/job-server/build.gradle | 3 +--
runners/spark/job-server/spark_job_server.gradle | 2 ++
runners/spark/spark_runner.gradle | 9 +++------
runners/twister2/build.gradle | 1 +
.../{UsesJavaExpansionService.java => UsesExternalService.java} | 7 ++++---
.../org/apache/beam/sdk/testing/UsesJavaExpansionService.java | 2 +-
.../org/apache/beam/sdk/testing/UsesPythonExpansionService.java | 2 +-
24 files changed, 25 insertions(+), 21 deletions(-)
diff --git a/.github/trigger_files/beam_PostCommit_Java_PVR_Spark3_Streaming.json b/.github/trigger_files/beam_PostCommit_Java_PVR_Spark3_Streaming.json
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/.github/trigger_files/beam_PostCommit_Java_PVR_Spark_Batch.json b/.github/trigger_files/beam_PostCommit_Java_PVR_Spark_Batch.json
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Dataflow_V2.json b/.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Dataflow_V2.json
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Dataflow_V2_Streaming.json b/.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Dataflow_V2_Streaming.json
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Flink_Java11.json b/.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Flink_Java11.json
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_SparkStructuredStreaming.json b/.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_SparkStructuredStreaming.json
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Twister2.json b/.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Twister2.json
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_ULR.json b/.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_ULR.json
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/.github/trigger_files/beam_PostCommit_XVR_Flink.json b/.github/trigger_files/beam_PostCommit_XVR_Flink.json
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
index 6166085fc8b..5b73940b99d 100644
--- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
+++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
@@ -2680,6 +2680,7 @@ class BeamModulePlugin implements Plugin<Project> {
project.evaluationDependsOn(":sdks:python")
project.evaluationDependsOn(":sdks:java:testing:expansion-service")
+ project.evaluationDependsOn(":sdks:java:core")
project.evaluationDependsOn(":sdks:java:extensions:python")
project.evaluationDependsOn(":sdks:go:test")
@@ -2744,9 +2745,11 @@ class BeamModulePlugin implements Plugin<Project> {
systemProperty "expansionPort", port
systemProperty "semiPersistDir", config.semiPersistDir
classpath = config.classpath + project.files(
+ project.project(":sdks:java:core").sourceSets.test.runtimeClasspath,
project.project(":sdks:java:extensions:python").sourceSets.test.runtimeClasspath
)
testClassesDirs = project.files(
+ project.project(":sdks:java:core").sourceSets.test.output.classesDirs,
project.project(":sdks:java:extensions:python").sourceSets.test.output.classesDirs
)
maxParallelForks config.numParallelTests
diff --git a/runners/direct-java/build.gradle b/runners/direct-java/build.gradle
index b7f74dc3e53..c357b8a0432 100644
--- a/runners/direct-java/build.gradle
+++ b/runners/direct-java/build.gradle
@@ -142,8 +142,7 @@ task needsRunnerTests(type: Test) {
excludeCategories "org.apache.beam.sdk.testing.LargeKeys\$Above100MB"
// MetricsPusher isn't implemented in direct runner
excludeCategories "org.apache.beam.sdk.testing.UsesMetricsPusher"
- excludeCategories "org.apache.beam.sdk.testing.UsesJavaExpansionService"
- excludeCategories "org.apache.beam.sdk.testing.UsesPythonExpansionService"
+ excludeCategories "org.apache.beam.sdk.testing.UsesExternalService"
excludeCategories 'org.apache.beam.sdk.testing.UsesBundleFinalizer'
}
testLogging {
@@ -173,8 +172,7 @@ task validatesRunner(type: Test) {
excludeCategories 'org.apache.beam.sdk.testing.UsesSdkHarnessEnvironment'
excludeCategories "org.apache.beam.sdk.testing.LargeKeys\$Above100MB"
excludeCategories 'org.apache.beam.sdk.testing.UsesMetricsPusher'
- excludeCategories "org.apache.beam.sdk.testing.UsesJavaExpansionService"
- excludeCategories "org.apache.beam.sdk.testing.UsesPythonExpansionService"
+ excludeCategories "org.apache.beam.sdk.testing.UsesExternalService"
// https://github.com/apache/beam/issues/18499
excludeCategories 'org.apache.beam.sdk.testing.UsesLoopingTimer'
}
diff --git a/runners/flink/flink_runner.gradle b/runners/flink/flink_runner.gradle
index d8bfcb3f533..1d91284a3d1 100644
--- a/runners/flink/flink_runner.gradle
+++ b/runners/flink/flink_runner.gradle
@@ -279,6 +279,7 @@ def createValidatesRunnerTask(Map m) {
excludeCategories 'org.apache.beam.sdk.testing.UsesTestStream'
} else {
includeCategories 'org.apache.beam.sdk.testing.ValidatesRunner'
+ excludeCategories 'org.apache.beam.sdk.testing.UsesExternalService'
// Should be run only in a properly configured SDK harness environment
excludeCategories 'org.apache.beam.sdk.testing.UsesSdkHarnessEnvironment'
excludeCategories 'org.apache.beam.sdk.testing.UsesBundleFinalizer'
diff --git a/runners/flink/job-server/flink_job_server.gradle b/runners/flink/job-server/flink_job_server.gradle
index ab3f726e6ad..e88b3c7d9e8 100644
--- a/runners/flink/job-server/flink_job_server.gradle
+++ b/runners/flink/job-server/flink_job_server.gradle
@@ -167,8 +167,7 @@ def portableValidatesRunnerTask(String name, boolean streaming, boolean checkpoi
// Larger keys are possible, but they require more memory.
excludeCategories 'org.apache.beam.sdk.testing.LargeKeys$Above10MB'
excludeCategories 'org.apache.beam.sdk.testing.UsesCommittedMetrics'
- excludeCategories 'org.apache.beam.sdk.testing.UsesJavaExpansionService'
- excludeCategories 'org.apache.beam.sdk.testing.UsesPythonExpansionService'
+ excludeCategories 'org.apache.beam.sdk.testing.UsesExternalService'
excludeCategories 'org.apache.beam.sdk.testing.UsesCustomWindowMerging'
excludeCategories 'org.apache.beam.sdk.testing.UsesFailureMessage'
excludeCategories 'org.apache.beam.sdk.testing.UsesGaugeMetrics'
diff --git a/runners/google-cloud-dataflow-java/build.gradle b/runners/google-cloud-dataflow-java/build.gradle
index 27b1ca83a9b..4c4b58d9032 100644
--- a/runners/google-cloud-dataflow-java/build.gradle
+++ b/runners/google-cloud-dataflow-java/build.gradle
@@ -180,8 +180,7 @@ def commonLegacyExcludeCategories = [
'org.apache.beam.sdk.testing.UsesSdkHarnessEnvironment',
'org.apache.beam.sdk.testing.LargeKeys$Above10MB',
'org.apache.beam.sdk.testing.UsesAttemptedMetrics',
- 'org.apache.beam.sdk.testing.UsesJavaExpansionService',
- 'org.apache.beam.sdk.testing.UsesPythonExpansionService',
+ 'org.apache.beam.sdk.testing.UsesExternalService',
'org.apache.beam.sdk.testing.UsesDistributionMetrics',
'org.apache.beam.sdk.testing.UsesGaugeMetrics',
'org.apache.beam.sdk.testing.UsesMultimapState',
@@ -192,6 +191,7 @@ def commonLegacyExcludeCategories = [
]
def commonRunnerV2ExcludeCategories = [
+ 'org.apache.beam.sdk.testing.UsesExternalService',
'org.apache.beam.sdk.testing.UsesGaugeMetrics',
'org.apache.beam.sdk.testing.UsesSetState',
'org.apache.beam.sdk.testing.UsesMapState',
diff --git a/runners/jet/build.gradle b/runners/jet/build.gradle
index 2b6bf2bcdff..56a001a2bce 100644
--- a/runners/jet/build.gradle
+++ b/runners/jet/build.gradle
@@ -71,6 +71,7 @@ task validatesRunnerBatch(type: Test) {
useJUnit {
includeCategories 'org.apache.beam.sdk.testing.ValidatesRunner'
// Should be run only in a properly configured SDK harness environment
+ excludeCategories 'org.apache.beam.sdk.testing.UsesExternalService'
excludeCategories 'org.apache.beam.sdk.testing.UsesSdkHarnessEnvironment'
excludeCategories "org.apache.beam.sdk.testing.LargeKeys\$Above100MB"
excludeCategories 'org.apache.beam.sdk.testing.UsesTimerMap'
diff --git a/runners/portability/java/build.gradle b/runners/portability/java/build.gradle
index 45709a0dbd1..9830b48c83a 100644
--- a/runners/portability/java/build.gradle
+++ b/runners/portability/java/build.gradle
@@ -149,6 +149,7 @@ def createUlrValidatesRunnerTask = { name, environmentType, dockerImageTask = ""
useJUnit {
includeCategories 'org.apache.beam.sdk.testing.ValidatesRunner'
// Should be run only in a properly configured SDK harness environment
+ excludeCategories 'org.apache.beam.sdk.testing.UsesExternalService'
excludeCategories 'org.apache.beam.sdk.testing.UsesSdkHarnessEnvironment'
excludeCategories 'org.apache.beam.sdk.testing.UsesGaugeMetrics'
excludeCategories 'org.apache.beam.sdk.testing.UsesOnWindowExpiration'
diff --git a/runners/samza/build.gradle b/runners/samza/build.gradle
index 8e2cc2a5eb7..a50e0d62e59 100644
--- a/runners/samza/build.gradle
+++ b/runners/samza/build.gradle
@@ -124,6 +124,7 @@ tasks.register("validatesRunner", Test) {
useJUnit {
includeCategories 'org.apache.beam.sdk.testing.NeedsRunner'
includeCategories 'org.apache.beam.sdk.testing.ValidatesRunner'
+ excludeCategories 'org.apache.beam.sdk.testing.UsesExternalService'
// Should be run only in a properly configured SDK harness environment
excludeCategories 'org.apache.beam.sdk.testing.UsesSdkHarnessEnvironment'
excludeCategories 'org.apache.beam.sdk.testing.UsesUnboundedSplittableParDo'
diff --git a/runners/samza/job-server/build.gradle b/runners/samza/job-server/build.gradle
index c9401a8aff1..7bb1b84dbaa 100644
--- a/runners/samza/job-server/build.gradle
+++ b/runners/samza/job-server/build.gradle
@@ -86,8 +86,7 @@ def portableValidatesRunnerTask(String name, boolean docker) {
// Larger keys are possible, but they require more memory.
excludeCategories 'org.apache.beam.sdk.testing.LargeKeys$Above10MB'
excludeCategories 'org.apache.beam.sdk.testing.UsesCommittedMetrics'
- excludeCategories 'org.apache.beam.sdk.testing.UsesJavaExpansionService'
- excludeCategories 'org.apache.beam.sdk.testing.UsesPythonExpansionService'
+ excludeCategories 'org.apache.beam.sdk.testing.UsesExternalService'
excludeCategories 'org.apache.beam.sdk.testing.UsesCustomWindowMerging'
excludeCategories 'org.apache.beam.sdk.testing.UsesFailureMessage'
excludeCategories 'org.apache.beam.sdk.testing.UsesGaugeMetrics'
diff --git a/runners/spark/job-server/spark_job_server.gradle b/runners/spark/job-server/spark_job_server.gradle
index 8945ad3498c..6c884e8e223 100644
--- a/runners/spark/job-server/spark_job_server.gradle
+++ b/runners/spark/job-server/spark_job_server.gradle
@@ -108,6 +108,7 @@ def portableValidatesRunnerTask(String name, boolean streaming, boolean docker,
testCategories = {
includeCategories 'org.apache.beam.sdk.testing.ValidatesRunner'
+ excludeCategories 'org.apache.beam.sdk.testing.UsesExternalService'
// Should be run only in a properly configured SDK harness environment
excludeCategories 'org.apache.beam.sdk.testing.UsesSdkHarnessEnvironment'
excludeCategories 'org.apache.beam.sdk.testing.FlattenWithHeterogeneousCoders'
@@ -167,6 +168,7 @@ def portableValidatesRunnerTask(String name, boolean streaming, boolean docker,
// Batch
testCategories = {
includeCategories 'org.apache.beam.sdk.testing.ValidatesRunner'
+ excludeCategories 'org.apache.beam.sdk.testing.UsesExternalService'
// Should be run only in a properly configured SDK harness environment
excludeCategories 'org.apache.beam.sdk.testing.UsesSdkHarnessEnvironment'
excludeCategories 'org.apache.beam.sdk.testing.FlattenWithHeterogeneousCoders'
diff --git a/runners/spark/spark_runner.gradle b/runners/spark/spark_runner.gradle
index d775cfd4d6e..5d0a7f02d17 100644
--- a/runners/spark/spark_runner.gradle
+++ b/runners/spark/spark_runner.gradle
@@ -274,8 +274,7 @@ def applyBatchValidatesRunnerSetup = { Test it ->
// SDF
excludeCategories 'org.apache.beam.sdk.testing.UsesUnboundedSplittableParDo'
// Portability
- excludeCategories 'org.apache.beam.sdk.testing.UsesJavaExpansionService'
- excludeCategories 'org.apache.beam.sdk.testing.UsesPythonExpansionService'
+ excludeCategories 'org.apache.beam.sdk.testing.UsesExternalService'
excludeCategories 'org.apache.beam.sdk.testing.UsesBundleFinalizer'
// Ordering
excludeCategories 'org.apache.beam.sdk.testing.UsesPerKeyOrderedDelivery'
@@ -349,8 +348,7 @@ def validatesRunnerStreaming = tasks.register("validatesRunnerStreaming", Test)
excludeCategories 'org.apache.beam.sdk.testing.UsesUnboundedSplittableParDo'
excludeCategories 'org.apache.beam.sdk.testing.UsesBoundedSplittableParDo'
// Portability
- excludeCategories 'org.apache.beam.sdk.testing.UsesJavaExpansionService'
- excludeCategories 'org.apache.beam.sdk.testing.UsesPythonExpansionService'
+ excludeCategories 'org.apache.beam.sdk.testing.UsesExternalService'
excludeCategories 'org.apache.beam.sdk.testing.UsesBundleFinalizer'
// Ordering
excludeCategories 'org.apache.beam.sdk.testing.UsesPerKeyOrderedDelivery'
@@ -405,8 +403,7 @@ tasks.register("validatesStructuredStreamingRunnerBatch", Test) {
// SDF
excludeCategories 'org.apache.beam.sdk.testing.UsesUnboundedSplittableParDo'
// Portability
- excludeCategories 'org.apache.beam.sdk.testing.UsesJavaExpansionService'
- excludeCategories 'org.apache.beam.sdk.testing.UsesPythonExpansionService'
+ excludeCategories 'org.apache.beam.sdk.testing.UsesExternalService'
excludeCategories 'org.apache.beam.sdk.testing.UsesBundleFinalizer'
excludeCategories 'org.apache.beam.sdk.testing.UsesTriggeredSideInputs'
}
diff --git a/runners/twister2/build.gradle b/runners/twister2/build.gradle
index 36a044737d8..744c4e171fd 100644
--- a/runners/twister2/build.gradle
+++ b/runners/twister2/build.gradle
@@ -80,6 +80,7 @@ def validatesRunnerBatch = tasks.register("validatesRunnerBatch", Test) {
forkEvery 1
useJUnit {
includeCategories 'org.apache.beam.sdk.testing.ValidatesRunner'
+ excludeCategories 'org.apache.beam.sdk.testing.UsesExternalService'
// Should be run only in a properly configured SDK harness environment
excludeCategories 'org.apache.beam.sdk.testing.UsesSdkHarnessEnvironment'
excludeCategories 'org.apache.beam.sdk.testing.FlattenWithHeterogeneousCoders'
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/UsesJavaExpansionService.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/UsesExternalService.java
similarity index 78%
copy from sdks/java/core/src/main/java/org/apache/beam/sdk/testing/UsesJavaExpansionService.java
copy to sdks/java/core/src/main/java/org/apache/beam/sdk/testing/UsesExternalService.java
index ad919ae4b88..a9e0b9d2236 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/UsesJavaExpansionService.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/UsesExternalService.java
@@ -20,8 +20,9 @@ package org.apache.beam.sdk.testing;
import org.apache.beam.sdk.annotations.Internal;
/**
- * Category tag for tests which use the expansion service in Java. Tests tagged with {@link
- * UsesJavaExpansionService} should be run for runners which support cross-language transforms.
+ * Category tag for tests which relies on a pre-defined port, such as expansion service or transform
+ * service. Tests tagged with {@link UsesExternalService} should initialize such port before the
+ * test execution.
*/
@Internal
-public interface UsesJavaExpansionService {}
+public interface UsesExternalService {}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/UsesJavaExpansionService.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/UsesJavaExpansionService.java
index ad919ae4b88..766854c8caf 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/UsesJavaExpansionService.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/UsesJavaExpansionService.java
@@ -24,4 +24,4 @@ import org.apache.beam.sdk.annotations.Internal;
* UsesJavaExpansionService} should be run for runners which support cross-language transforms.
*/
@Internal
-public interface UsesJavaExpansionService {}
+public interface UsesJavaExpansionService extends UsesExternalService {}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/UsesPythonExpansionService.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/UsesPythonExpansionService.java
index b92742e5db8..0fbab223934 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/UsesPythonExpansionService.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/UsesPythonExpansionService.java
@@ -24,4 +24,4 @@ import org.apache.beam.sdk.annotations.Internal;
* UsesPythonExpansionService} should be run for runners which support cross-language transforms.
*/
@Internal
-public interface UsesPythonExpansionService {}
+public interface UsesPythonExpansionService extends UsesExternalService {}