You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2021/11/14 16:47:01 UTC
[spark] branch branch-3.2 updated: [SPARK-37320][K8S][TESTS] Delete py_container_checks.zip after the test in DepsTestsSuite finishes
This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new 96c17d3 [SPARK-37320][K8S][TESTS] Delete py_container_checks.zip after the test in DepsTestsSuite finishes
96c17d3 is described below
commit 96c17d31ff85d2e2be5ddaf51359421a6ac027a0
Author: Kousuke Saruta <sa...@oss.nttdata.com>
AuthorDate: Sun Nov 14 08:45:20 2021 -0800
[SPARK-37320][K8S][TESTS] Delete py_container_checks.zip after the test in DepsTestsSuite finishes
### What changes were proposed in this pull request?
This PR fixes an issue that `py_container_checks.zip` still remains in `resource-managers/kubernetes/integration-tests/tests/` even after the test `Launcher python client dependencies using a zip file` in `DepsTestsSuite` finishes.
### Why are the changes needed?
To keep the repository clean.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Confirmed that the zip file will be removed after the test finishes with the following command using MiniKube.
```
PVC_TESTS_HOST_PATH=/path PVC_TESTS_VM_PATH=/path build/mvn -Dspark.kubernetes.test.namespace=default -Pkubernetes -Pkubernetes-integration-tests -pl resource-managers/kubernetes/integration-tests integration-test
```
Closes #34588 from sarutak/remove-zip-k8s.
Authored-by: Kousuke Saruta <sa...@oss.nttdata.com>
Signed-off-by: Dongjoon Hyun <do...@apache.org>
(cherry picked from commit d8cee85e0264fe879f9d1eeec7541a8e94ff83f6)
Signed-off-by: Dongjoon Hyun <do...@apache.org>
---
.../k8s/integrationtest/DepsTestsSuite.scala | 22 ++++++++++++++--------
1 file changed, 14 insertions(+), 8 deletions(-)
diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/DepsTestsSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/DepsTestsSuite.scala
index 812fd21..3f3c4ef 100644
--- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/DepsTestsSuite.scala
+++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/DepsTestsSuite.scala
@@ -16,7 +16,9 @@
*/
package org.apache.spark.deploy.k8s.integrationtest
+import java.io.File
import java.net.URL
+import java.nio.file.Files
import scala.collection.JavaConverters._
@@ -223,14 +225,18 @@ private[spark] trait DepsTestsSuite { k8sSuite: KubernetesSuite =>
val pySparkFiles = Utils.getTestFileAbsolutePath("pyfiles.py", sparkHomeDir)
val inDepsFile = Utils.getTestFileAbsolutePath("py_container_checks.py", sparkHomeDir)
val outDepsFile = s"${inDepsFile.substring(0, inDepsFile.lastIndexOf("."))}.zip"
- Utils.createZipFile(inDepsFile, outDepsFile)
- testPython(
- pySparkFiles,
- Seq(
- "Python runtime version check is: True",
- "Python environment version check is: True",
- "Python runtime version check for executor is: True"),
- Some(outDepsFile))
+ try {
+ Utils.createZipFile(inDepsFile, outDepsFile)
+ testPython(
+ pySparkFiles,
+ Seq(
+ "Python runtime version check is: True",
+ "Python environment version check is: True",
+ "Python runtime version check for executor is: True"),
+ Some(outDepsFile))
+ } finally {
+ Files.delete(new File(outDepsFile).toPath)
+ }
}
private def testPython(
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org