[beam] 01/01: Add flag to do a docker system prune between docker push steps (#26967)

[da...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

damccorm pushed a commit to branch users/damccorm/dockerprune-cp
in repository https://gitbox.apache.org/repos/asf/beam.git

commit 06437ea8ef4bf8b6a1cb3492fdabb82605212ac4
Author: Danny McCormick <da...@google.com>
AuthorDate: Fri Jun 30 08:46:53 2023 -0400

    Add flag to do a docker system prune between docker push steps (#26967)
    
    * Add flag to do a docker system prune between docker push steps
    
    * Push to my docker image
    
    * order/prune python container pushes
    
    * Prune docker between pushes
    
    * Consolidate to loop
    
    * Temporarily point to my dockerhub to test/confirm
    
    * restore apache as dockerhub repo root
    
    * whitespace
---
 build.gradle.kts                   | 53 +++++++++++++++++++++++++++++++++++---
 sdks/python/container/build.gradle | 30 ++++++++++++++++++---
 2 files changed, 76 insertions(+), 7 deletions(-)

diff --git a/build.gradle.kts b/build.gradle.kts
index 6ba7ebeaca4..57438fa6768 100644
--- a/build.gradle.kts
+++ b/build.gradle.kts
@@ -573,18 +573,65 @@ tasks.register("typescriptPreCommit") {
   dependsOn(":sdks:python:test-suites:tox:py38:jest")
 }
 
-tasks.register("pushAllDockerImages") {
+tasks.register("pushAllRunnersDockerImages") {
   dependsOn(":runners:spark:3:job-server:container:dockerPush")
+  for (version in project.ext.get("allFlinkVersions") as Array<*>) {
+    dependsOn(":runners:flink:${version}:job-server-container:dockerPush")
+  }
+  
+  doLast {
+    if (project.hasProperty("prune-images")) {
+      exec {
+        executable("docker")
+        args("system", "prune", "-a", "--force")
+      }
+    }
+  }
+}
+
+tasks.register("pushAllSdkDockerImages") {
+  // Enforce ordering to allow the prune step to happen between runs.
+  // This will ensure we don't use up too much space (especially in CI environments)
+  mustRunAfter(":pushAllRunnersDockerImages")
+
   dependsOn(":sdks:java:container:pushAll")
   dependsOn(":sdks:python:container:pushAll")
   dependsOn(":sdks:go:container:pushAll")
   dependsOn(":sdks:typescript:container:pushAll")
-  for (version in project.ext.get("allFlinkVersions") as Array<*>) {
-    dependsOn(":runners:flink:${version}:job-server-container:dockerPush")
+  
+  doLast {
+    if (project.hasProperty("prune-images")) {
+      exec {
+        executable("docker")
+        args("system", "prune", "-a", "--force")
+      }
+    }
   }
+}
+
+tasks.register("pushAllXlangDockerImages") {
+  // Enforce ordering to allow the prune step to happen between runs.
+  // This will ensure we don't use up too much space (especially in CI environments)
+  mustRunAfter(":pushAllSdkDockerImages")
+
   dependsOn(":sdks:java:expansion-service:container:dockerPush")
   dependsOn(":sdks:java:transform-service:controller-container:dockerPush")
   dependsOn(":sdks:python:expansion-service-container:dockerPush")
+  
+  doLast {
+    if (project.hasProperty("prune-images")) {
+      exec {
+        executable("docker")
+        args("system", "prune", "-a", "--force")
+      }
+    }
+  }
+}
+
+tasks.register("pushAllDockerImages") {
+  dependsOn(":pushAllRunnersDockerImages")
+  dependsOn(":pushAllSdkDockerImages")
+  dependsOn(":pushAllXlangDockerImages")
 }
 
 // Use this task to validate the environment set up for Go, Python and Java
diff --git a/sdks/python/container/build.gradle b/sdks/python/container/build.gradle
index 96a9c17b9a2..f6b31eec5b9 100644
--- a/sdks/python/container/build.gradle
+++ b/sdks/python/container/build.gradle
@@ -42,11 +42,33 @@ tasks.register("buildAll") {
   dependsOn ':sdks:python:container:py311:docker'
 }
 
+for(int i=8; i<=11; ++i) {
+  String cur = "3" + i
+  String prev = "3" + (i-1)
+  tasks.register("push" + cur) {
+    if (cur != "38") {
+      // Enforce ordering to allow the prune step to happen between runs.
+      // This will ensure we don't use up too much space (especially in CI environments)
+      mustRunAfter(":sdks:python:container:push" + prev)
+    }
+    dependsOn ':sdks:python:container:py' + cur + ':dockerPush'
+
+    doLast {
+      if (project.hasProperty("prune-images")) {
+        exec {
+          executable("docker")
+          args("system", "prune", "-a", "--force")
+        }
+      }
+    }
+  }
+}
+
 tasks.register("pushAll") {
-  dependsOn ':sdks:python:container:py38:dockerPush'
-  dependsOn ':sdks:python:container:py39:dockerPush'
-  dependsOn ':sdks:python:container:py310:dockerPush'
-  dependsOn ':sdks:python:container:py311:dockerPush'
+  dependsOn ':sdks:python:container:push38'
+  dependsOn ':sdks:python:container:push39'
+  dependsOn ':sdks:python:container:push310'
+  dependsOn ':sdks:python:container:push311'
 }
 
 tasks.register("generatePythonRequirementsAll") {