[beam] branch users/damccorm/dockerprune-cp created (now 06437ea8ef4)

[da...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

damccorm pushed a change to branch users/damccorm/dockerprune-cp
in repository https://gitbox.apache.org/repos/asf/beam.git


      at 06437ea8ef4 Add flag to do a docker system prune between docker push steps (#26967)

This branch includes the following new commits:

     new 06437ea8ef4 Add flag to do a docker system prune between docker push steps (#26967)

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.

[beam] 01/01: Add flag to do a docker system prune between docker push steps (#26967)

[da...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

damccorm pushed a commit to branch users/damccorm/dockerprune-cp
in repository https://gitbox.apache.org/repos/asf/beam.git

commit 06437ea8ef4bf8b6a1cb3492fdabb82605212ac4
Author: Danny McCormick <da...@google.com>
AuthorDate: Fri Jun 30 08:46:53 2023 -0400

    Add flag to do a docker system prune between docker push steps (#26967)
    
    * Add flag to do a docker system prune between docker push steps
    
    * Push to my docker image
    
    * order/prune python container pushes
    
    * Prune docker between pushes
    
    * Consolidate to loop
    
    * Temporarily point to my dockerhub to test/confirm
    
    * restore apache as dockerhub repo root
    
    * whitespace
---
 build.gradle.kts                   | 53 +++++++++++++++++++++++++++++++++++---
 sdks/python/container/build.gradle | 30 ++++++++++++++++++---
 2 files changed, 76 insertions(+), 7 deletions(-)

diff --git a/build.gradle.kts b/build.gradle.kts
index 6ba7ebeaca4..57438fa6768 100644
--- a/build.gradle.kts
+++ b/build.gradle.kts
@@ -573,18 +573,65 @@ tasks.register("typescriptPreCommit") {
   dependsOn(":sdks:python:test-suites:tox:py38:jest")
 }
 
-tasks.register("pushAllDockerImages") {
+tasks.register("pushAllRunnersDockerImages") {
   dependsOn(":runners:spark:3:job-server:container:dockerPush")
+  for (version in project.ext.get("allFlinkVersions") as Array<*>) {
+    dependsOn(":runners:flink:${version}:job-server-container:dockerPush")
+  }
+  
+  doLast {
+    if (project.hasProperty("prune-images")) {
+      exec {
+        executable("docker")
+        args("system", "prune", "-a", "--force")
+      }
+    }
+  }
+}
+
+tasks.register("pushAllSdkDockerImages") {
+  // Enforce ordering to allow the prune step to happen between runs.
+  // This will ensure we don't use up too much space (especially in CI environments)
+  mustRunAfter(":pushAllRunnersDockerImages")
+
   dependsOn(":sdks:java:container:pushAll")
   dependsOn(":sdks:python:container:pushAll")
   dependsOn(":sdks:go:container:pushAll")
   dependsOn(":sdks:typescript:container:pushAll")
-  for (version in project.ext.get("allFlinkVersions") as Array<*>) {
-    dependsOn(":runners:flink:${version}:job-server-container:dockerPush")
+  
+  doLast {
+    if (project.hasProperty("prune-images")) {
+      exec {
+        executable("docker")
+        args("system", "prune", "-a", "--force")
+      }
+    }
   }
+}
+
+tasks.register("pushAllXlangDockerImages") {
+  // Enforce ordering to allow the prune step to happen between runs.
+  // This will ensure we don't use up too much space (especially in CI environments)
+  mustRunAfter(":pushAllSdkDockerImages")
+
   dependsOn(":sdks:java:expansion-service:container:dockerPush")
   dependsOn(":sdks:java:transform-service:controller-container:dockerPush")
   dependsOn(":sdks:python:expansion-service-container:dockerPush")
+  
+  doLast {
+    if (project.hasProperty("prune-images")) {
+      exec {
+        executable("docker")
+        args("system", "prune", "-a", "--force")
+      }
+    }
+  }
+}
+
+tasks.register("pushAllDockerImages") {
+  dependsOn(":pushAllRunnersDockerImages")
+  dependsOn(":pushAllSdkDockerImages")
+  dependsOn(":pushAllXlangDockerImages")
 }
 
 // Use this task to validate the environment set up for Go, Python and Java
diff --git a/sdks/python/container/build.gradle b/sdks/python/container/build.gradle
index 96a9c17b9a2..f6b31eec5b9 100644
--- a/sdks/python/container/build.gradle
+++ b/sdks/python/container/build.gradle
@@ -42,11 +42,33 @@ tasks.register("buildAll") {
   dependsOn ':sdks:python:container:py311:docker'
 }
 
+for(int i=8; i<=11; ++i) {
+  String cur = "3" + i
+  String prev = "3" + (i-1)
+  tasks.register("push" + cur) {
+    if (cur != "38") {
+      // Enforce ordering to allow the prune step to happen between runs.
+      // This will ensure we don't use up too much space (especially in CI environments)
+      mustRunAfter(":sdks:python:container:push" + prev)
+    }
+    dependsOn ':sdks:python:container:py' + cur + ':dockerPush'
+
+    doLast {
+      if (project.hasProperty("prune-images")) {
+        exec {
+          executable("docker")
+          args("system", "prune", "-a", "--force")
+        }
+      }
+    }
+  }
+}
+
 tasks.register("pushAll") {
-  dependsOn ':sdks:python:container:py38:dockerPush'
-  dependsOn ':sdks:python:container:py39:dockerPush'
-  dependsOn ':sdks:python:container:py310:dockerPush'
-  dependsOn ':sdks:python:container:py311:dockerPush'
+  dependsOn ':sdks:python:container:push38'
+  dependsOn ':sdks:python:container:push39'
+  dependsOn ':sdks:python:container:push310'
+  dependsOn ':sdks:python:container:push311'
 }
 
 tasks.register("generatePythonRequirementsAll") {