You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by da...@apache.org on 2023/07/07 17:10:30 UTC
[beam] branch release-2.49.0 updated: Add flag to do a docker system prune between docker push steps (#26967) (#27391)
This is an automated email from the ASF dual-hosted git repository.
damccorm pushed a commit to branch release-2.49.0
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/release-2.49.0 by this push:
new b2b01d0ed4f Add flag to do a docker system prune between docker push steps (#26967) (#27391)
b2b01d0ed4f is described below
commit b2b01d0ed4f0ab273a44dcab5f869470fe166317
Author: Danny McCormick <da...@google.com>
AuthorDate: Fri Jul 7 13:10:24 2023 -0400
Add flag to do a docker system prune between docker push steps (#26967) (#27391)
* Add flag to do a docker system prune between docker push steps
* Push to my docker image
* order/prune python container pushes
* Prune docker between pushes
* Consolidate to loop
* Temporarily point to my dockerhub to test/confirm
* restore apache as dockerhub repo root
* whitespace
---
build.gradle.kts | 53 +++++++++++++++++++++++++++++++++++---
sdks/python/container/build.gradle | 30 ++++++++++++++++++---
2 files changed, 76 insertions(+), 7 deletions(-)
diff --git a/build.gradle.kts b/build.gradle.kts
index 6ba7ebeaca4..57438fa6768 100644
--- a/build.gradle.kts
+++ b/build.gradle.kts
@@ -573,18 +573,65 @@ tasks.register("typescriptPreCommit") {
dependsOn(":sdks:python:test-suites:tox:py38:jest")
}
-tasks.register("pushAllDockerImages") {
+tasks.register("pushAllRunnersDockerImages") {
dependsOn(":runners:spark:3:job-server:container:dockerPush")
+ for (version in project.ext.get("allFlinkVersions") as Array<*>) {
+ dependsOn(":runners:flink:${version}:job-server-container:dockerPush")
+ }
+
+ doLast {
+ if (project.hasProperty("prune-images")) {
+ exec {
+ executable("docker")
+ args("system", "prune", "-a", "--force")
+ }
+ }
+ }
+}
+
+tasks.register("pushAllSdkDockerImages") {
+ // Enforce ordering to allow the prune step to happen between runs.
+ // This will ensure we don't use up too much space (especially in CI environments)
+ mustRunAfter(":pushAllRunnersDockerImages")
+
dependsOn(":sdks:java:container:pushAll")
dependsOn(":sdks:python:container:pushAll")
dependsOn(":sdks:go:container:pushAll")
dependsOn(":sdks:typescript:container:pushAll")
- for (version in project.ext.get("allFlinkVersions") as Array<*>) {
- dependsOn(":runners:flink:${version}:job-server-container:dockerPush")
+
+ doLast {
+ if (project.hasProperty("prune-images")) {
+ exec {
+ executable("docker")
+ args("system", "prune", "-a", "--force")
+ }
+ }
}
+}
+
+tasks.register("pushAllXlangDockerImages") {
+ // Enforce ordering to allow the prune step to happen between runs.
+ // This will ensure we don't use up too much space (especially in CI environments)
+ mustRunAfter(":pushAllSdkDockerImages")
+
dependsOn(":sdks:java:expansion-service:container:dockerPush")
dependsOn(":sdks:java:transform-service:controller-container:dockerPush")
dependsOn(":sdks:python:expansion-service-container:dockerPush")
+
+ doLast {
+ if (project.hasProperty("prune-images")) {
+ exec {
+ executable("docker")
+ args("system", "prune", "-a", "--force")
+ }
+ }
+ }
+}
+
+tasks.register("pushAllDockerImages") {
+ dependsOn(":pushAllRunnersDockerImages")
+ dependsOn(":pushAllSdkDockerImages")
+ dependsOn(":pushAllXlangDockerImages")
}
// Use this task to validate the environment set up for Go, Python and Java
diff --git a/sdks/python/container/build.gradle b/sdks/python/container/build.gradle
index 96a9c17b9a2..f6b31eec5b9 100644
--- a/sdks/python/container/build.gradle
+++ b/sdks/python/container/build.gradle
@@ -42,11 +42,33 @@ tasks.register("buildAll") {
dependsOn ':sdks:python:container:py311:docker'
}
+for(int i=8; i<=11; ++i) {
+ String cur = "3" + i
+ String prev = "3" + (i-1)
+ tasks.register("push" + cur) {
+ if (cur != "38") {
+ // Enforce ordering to allow the prune step to happen between runs.
+ // This will ensure we don't use up too much space (especially in CI environments)
+ mustRunAfter(":sdks:python:container:push" + prev)
+ }
+ dependsOn ':sdks:python:container:py' + cur + ':dockerPush'
+
+ doLast {
+ if (project.hasProperty("prune-images")) {
+ exec {
+ executable("docker")
+ args("system", "prune", "-a", "--force")
+ }
+ }
+ }
+ }
+}
+
tasks.register("pushAll") {
- dependsOn ':sdks:python:container:py38:dockerPush'
- dependsOn ':sdks:python:container:py39:dockerPush'
- dependsOn ':sdks:python:container:py310:dockerPush'
- dependsOn ':sdks:python:container:py311:dockerPush'
+ dependsOn ':sdks:python:container:push38'
+ dependsOn ':sdks:python:container:push39'
+ dependsOn ':sdks:python:container:push310'
+ dependsOn ':sdks:python:container:push311'
}
tasks.register("generatePythonRequirementsAll") {