[beam] branch master updated: [BEAM-12830] Avoid flock and prepare task once. (OSX fix.) (#16333)

[lo...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

lostluck pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/master by this push:
     new af341e5  [BEAM-12830] Avoid flock and prepare task once. (OSX fix.) (#16333)
af341e5 is described below

commit af341e5c085d6df8dc36f8b0718026f45f3870b1
Author: Robert Burke <lo...@users.noreply.github.com>
AuthorDate: Wed Dec 22 17:41:07 2021 -0800

    [BEAM-12830] Avoid flock and prepare task once. (OSX fix.) (#16333)
---
 .../org/apache/beam/gradle/BeamModulePlugin.groovy | 17 +++++++-
 ...un_with_go_version.sh => prepare_go_version.sh} | 29 +++++--------
 sdks/go/run_with_go_version.sh                     | 47 +++++++++++++++-------
 3 files changed, 60 insertions(+), 33 deletions(-)

diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
index a91d62e..63727b4 100644
--- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
+++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
@@ -1812,9 +1812,24 @@ class BeamModulePlugin implements Plugin<Project> {
       project.check.dependsOn project.test
 
       def goRootDir = "${project.rootDir}/sdks/go"
-      project.ext.goCmd = "${goRootDir}/run_with_go_version.sh"
+
+      // This sets the whole project Go version.
+      project.ext.goVersion = "go1.16.12"
+
+      // Minor TODO: Figure out if we can pull out the GOCMD env variable after goPrepare script
+      // completion, and avoid this GOBIN substitution.
+      project.ext.goCmd = "${goRootDir}/run_with_go_version.sh --gocmd GOBIN/${project.ext.goVersion}"
+
+      project.tasks.create(name: "goPrepare") {
+        description "Prepare ${project.ext.goVersion} for builds and tests."
+        project.exec {
+          executable 'sh'
+          args '-c', "${goRootDir}/prepare_go_version.sh --version ${project.ext.goVersion}"
+        }
+      }
 
       project.tasks.create(name: "goBuild") {
+        dependsOn ":sdks:go:goPrepare"
         ext.goTargets = './...'
         ext.outputLocation = './build/bin/${GOOS}_${GOARCH}/'
         doLast {
diff --git a/sdks/go/run_with_go_version.sh b/sdks/go/prepare_go_version.sh
similarity index 68%
copy from sdks/go/run_with_go_version.sh
copy to sdks/go/prepare_go_version.sh
index cf1038f..d156f33 100755
--- a/sdks/go/run_with_go_version.sh
+++ b/sdks/go/prepare_go_version.sh
@@ -16,14 +16,10 @@
 #    limitations under the License.
 
 # This script sets the go version used by all Beam SDK scripts.
-# It requires an existing Go installation on the system, which
+# It requires an existing Go installation 1.16 or greater on the system, which
 # will be used to download specific versions of Go.
 #
-# The Go installation will use the local host platform, while the actual
-# go command will use the set GOOS and GOARCH env variables.
-#
-# Accepts the following optional flags, after which all following parameters
-# will be provided to the go version tool.
+# Accepts the following require flag:
 #    --version -> A string for a fully qualified go version, eg go1.16.5 or go1.18beta1
 #        The list of available versions are at https://go.dev/dl/ 
 
@@ -34,7 +30,7 @@ set -e
 #
 # This variable is also used as the execution command downscript.
 # The list of downloadable versions are at https://go.dev/dl/ 
-GOVERS=go1.16.12
+GOVERS="invalid"
 
 if ! command -v go &> /dev/null
 then
@@ -51,8 +47,9 @@ case $key in
         shift # past argument
         shift # past value
         ;;
-    *)  # unknown options are go tool args.
-        break
+    *)  # unknown args
+        echo "prepare_go_version requires the --version flag. See https://go.dev/dl/ for available versions."
+        exit 1
         ;;
 esac
 done
@@ -68,13 +65,9 @@ echo "System Go installation: `which go` is `go version`; Preparing to use $GOBI
 # Ensure it's installed in the GOBIN directory, using the local host platform.
 GOOS=$GOHOSTOS GOARCH=$GOHOSTARCH GOBIN=$GOBIN go install golang.org/dl/$GOVERS@latest
 
-LOCKFILE=$GOBIN/$GOVERS.lock
-# The download command isn't concurrency safe so we get an exclusive lock, without wait.
-# If we're first, we ensure the command is downloaded, releasing the lock afterwards.
-# This operation is cached on system and won't be re-downloaded at least.
-flock --exclusive --nonblock --conflict-exit-code 0 $LOCKFILE $GOBIN/$GOVERS download
+# The download command isn't concurrency safe so prepare should be done at most once
+# per gogradle chain.
+$GOBIN/$GOVERS download
 
-# Execute the script with the remaining arguments.
-# We get a shared lock for the ordinary go command execution.
-echo $GOBIN/$GOVERS $@
-flock --shared --timeout=10 $LOCKFILE $GOBIN/$GOVERS $@
+export GOCMD=$GOBIN/$GOVERS
+echo "GOCMD=$GOCMD"
diff --git a/sdks/go/run_with_go_version.sh b/sdks/go/run_with_go_version.sh
index cf1038f..0b2ce43 100755
--- a/sdks/go/run_with_go_version.sh
+++ b/sdks/go/run_with_go_version.sh
@@ -16,7 +16,7 @@
 #    limitations under the License.
 
 # This script sets the go version used by all Beam SDK scripts.
-# It requires an existing Go installation on the system, which
+# It requires an existing Go installation greater than 1.16 on the system, which
 # will be used to download specific versions of Go.
 #
 # The Go installation will use the local host platform, while the actual
@@ -26,6 +26,9 @@
 # will be provided to the go version tool.
 #    --version -> A string for a fully qualified go version, eg go1.16.5 or go1.18beta1
 #        The list of available versions are at https://go.dev/dl/ 
+#    --gocmd -> a specific path to a Go command to execute. If present, ignores --version flag
+#        and avoids doing the download check step.
+
 
 
 set -e
@@ -51,6 +54,11 @@ case $key in
         shift # past argument
         shift # past value
         ;;
+    --gocmd)
+        GOCMD="$2"
+        shift # past argument
+        shift # past value
+        ;;
     *)  # unknown options are go tool args.
         break
         ;;
@@ -62,19 +70,30 @@ GOBIN=$GOPATH/bin
 GOHOSTOS=`go env GOHOSTOS`
 GOHOSTARCH=`go env GOHOSTARCH`
 
-# Outputing the system Go version for debugging purposes.
-echo "System Go installation: `which go` is `go version`; Preparing to use $GOBIN/$GOVERS"
 
-# Ensure it's installed in the GOBIN directory, using the local host platform.
-GOOS=$GOHOSTOS GOARCH=$GOHOSTARCH GOBIN=$GOBIN go install golang.org/dl/$GOVERS@latest
+# Check if we've already prepared the Go command. If so, then we don't need to
+# do the download and versioning check.
+if [ -z "$GOCMD" ] ; then
+    # Outputing the system Go version for debugging purposes.
+    echo "System Go installation: `which go` is `go version`; Preparing to use $GOBIN/$GOVERS"
+    # Ensure it's installed in the GOBIN directory, using the local host platform.
+    GOOS=$GOHOSTOS GOARCH=$GOHOSTARCH GOBIN=$GOBIN go install golang.org/dl/$GOVERS@latest
+
+    LOCKFILE=$GOBIN/$GOVERS.lock
+    # The download command isn't concurrency safe so we get an exclusive lock, without wait.
+    # If we're first, we ensure the command is downloaded, releasing the lock afterwards.
+    # This operation is cached on system and won't be re-downloaded at least.
+    flock --exclusive --nonblock --conflict-exit-code 0 $LOCKFILE $GOBIN/$GOVERS download
 
-LOCKFILE=$GOBIN/$GOVERS.lock
-# The download command isn't concurrency safe so we get an exclusive lock, without wait.
-# If we're first, we ensure the command is downloaded, releasing the lock afterwards.
-# This operation is cached on system and won't be re-downloaded at least.
-flock --exclusive --nonblock --conflict-exit-code 0 $LOCKFILE $GOBIN/$GOVERS download
+    # Execute the script with the remaining arguments.
+    # We get a shared lock for the ordinary go command execution.
+    echo $GOBIN/$GOVERS $@
+    flock --shared --timeout=10 $LOCKFILE $GOBIN/$GOVERS $@
+else
+    # Minor TODO: Figure out if we can pull out the GOCMD env variable after goPrepare
+    # completion, and avoid this brittle GOBIN substitution.
+    GOCMD=${GOCMD/GOBIN/$GOBIN}
 
-# Execute the script with the remaining arguments.
-# We get a shared lock for the ordinary go command execution.
-echo $GOBIN/$GOVERS $@
-flock --shared --timeout=10 $LOCKFILE $GOBIN/$GOVERS $@
+    echo $GOCMD $@
+    $GOCMD $@
+fi