[beam] 01/01: Add arm tests to postcommit suite

[da...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

damccorm pushed a commit to branch users/damccorm/armPostCommitIT
in repository https://gitbox.apache.org/repos/asf/beam.git

commit d776cc5f17cdb1f27351fad4a6de9dbd74809124
Author: Danny McCormick <da...@google.com>
AuthorDate: Mon Sep 25 10:22:52 2023 -0400

    Add arm tests to postcommit suite
---
 build.gradle.kts                               |  4 ++++
 sdks/python/scripts/run_integration_test.sh    | 10 ++++++++++
 sdks/python/test-suites/dataflow/common.gradle | 20 ++++++++++++++++++++
 3 files changed, 34 insertions(+)

diff --git a/build.gradle.kts b/build.gradle.kts
index a4a9a09e504..438e1b9e874 100644
--- a/build.gradle.kts
+++ b/build.gradle.kts
@@ -496,6 +496,7 @@ tasks.register("pythonFormatterPreCommit") {
 
 tasks.register("python38PostCommit") {
   dependsOn(":sdks:python:test-suites:dataflow:py38:postCommitIT")
+  dependsOn(":sdks:python:test-suites:dataflow:py38:postCommitArmIT")
   dependsOn(":sdks:python:test-suites:direct:py38:postCommitIT")
   dependsOn(":sdks:python:test-suites:direct:py38:hdfsIntegrationTest")
   dependsOn(":sdks:python:test-suites:portable:py38:postCommitPy38")
@@ -509,6 +510,7 @@ tasks.register("python38PostCommit") {
 
 tasks.register("python39PostCommit") {
   dependsOn(":sdks:python:test-suites:dataflow:py39:postCommitIT")
+  dependsOn(":sdks:python:test-suites:dataflow:py39:postCommitArmIT")
   dependsOn(":sdks:python:test-suites:direct:py39:postCommitIT")
   dependsOn(":sdks:python:test-suites:direct:py39:hdfsIntegrationTest")
   dependsOn(":sdks:python:test-suites:portable:py39:postCommitPy39")
@@ -519,6 +521,7 @@ tasks.register("python39PostCommit") {
 
 tasks.register("python310PostCommit") {
   dependsOn(":sdks:python:test-suites:dataflow:py310:postCommitIT")
+  dependsOn(":sdks:python:test-suites:dataflow:py310:postCommitArmIT")
   dependsOn(":sdks:python:test-suites:direct:py310:postCommitIT")
   dependsOn(":sdks:python:test-suites:direct:py310:hdfsIntegrationTest")
   dependsOn(":sdks:python:test-suites:portable:py310:postCommitPy310")
@@ -526,6 +529,7 @@ tasks.register("python310PostCommit") {
 
 tasks.register("python311PostCommit") {
   dependsOn(":sdks:python:test-suites:dataflow:py311:postCommitIT")
+  dependsOn(":sdks:python:test-suites:dataflow:py311:postCommitArmIT")
   dependsOn(":sdks:python:test-suites:direct:py311:postCommitIT")
   dependsOn(":sdks:python:test-suites:direct:py311:hdfsIntegrationTest")
   dependsOn(":sdks:python:test-suites:portable:py311:postCommitPy311")
diff --git a/sdks/python/scripts/run_integration_test.sh b/sdks/python/scripts/run_integration_test.sh
index 4f29ed5a4ad..6ad592080ae 100755
--- a/sdks/python/scripts/run_integration_test.sh
+++ b/sdks/python/scripts/run_integration_test.sh
@@ -78,6 +78,7 @@ KMS_KEY_NAME="projects/apache-beam-testing/locations/global/keyRings/beam-it/cry
 SUITE=""
 COLLECT_MARKERS=
 REQUIREMENTS_FILE=""
+ARCH=""
 
 # Default test (pytest) options.
 # Run WordCountIT.test_wordcount_it by default if no test options are
@@ -163,6 +164,11 @@ case $key in
       shift # past argument
       shift # past value
       ;;
+    --arch)
+      ARCH="$2"
+      shift # past argument
+      shift # past value
+      ;;
     *)    # unknown option
         echo "Unknown option: $1"
         exit 1
@@ -234,6 +240,10 @@ if [[ -z $PIPELINE_OPTS ]]; then
     opts+=("--streaming")
   fi
 
+  if [[ "$ARCH" == "ARM" ]]; then
+    opts+=("--machine_type=t2a-standard-1")
+  fi
+
   if [[ ! -z "$KMS_KEY_NAME" ]]; then
     opts+=(
       "--kms_key_name=$KMS_KEY_NAME"
diff --git a/sdks/python/test-suites/dataflow/common.gradle b/sdks/python/test-suites/dataflow/common.gradle
index 8b8a5680899..6b09f7d27e5 100644
--- a/sdks/python/test-suites/dataflow/common.gradle
+++ b/sdks/python/test-suites/dataflow/common.gradle
@@ -143,6 +143,26 @@ task postCommitIT {
   }
 }
 
+task postCommitArmIT {
+  dependsOn 'initializeForDataflowJob'
+
+  doLast {
+    def testOpts = basicPytestOpts + ["--numprocesses=8", "--dist=loadfile"]
+    def argMap = [
+        "test_opts": testOpts,
+        "sdk_location": project.ext.sdkLocation,
+        "suite": "postCommitIT-df${pythonVersionSuffix}",
+        "collect": "it_postcommit",
+        "arch": "ARM"
+    ]
+    def cmdArgs = mapToArgString(argMap)
+    exec {
+      executable 'sh'
+      args '-c', ". ${envdir}/bin/activate && ${runScriptsDir}/run_integration_test.sh $cmdArgs"
+    }
+  }
+}
+
 task postCommitSickbay {
   dependsOn 'initializeForDataflowJob'