You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemds.apache.org by ba...@apache.org on 2021/09/25 20:16:35 UTC
[systemds] branch master updated: [MINOR] Add spark checkpoint dir
for local execution
This is an automated email from the ASF dual-hosted git repository.
baunsgaard pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/systemds.git
The following commit(s) were added to refs/heads/master by this push:
new 39e8f31 [MINOR] Add spark checkpoint dir for local execution
39e8f31 is described below
commit 39e8f31e1b5a5bf4c164dfe09e07b6416880761a
Author: baunsgaard <ba...@tugraz.at>
AuthorDate: Sat Sep 25 22:16:23 2021 +0200
[MINOR] Add spark checkpoint dir for local execution
---
.../sysds/runtime/controlprogram/context/SparkExecutionContext.java | 3 +++
1 file changed, 3 insertions(+)
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/context/SparkExecutionContext.java b/src/main/java/org/apache/sysds/runtime/controlprogram/context/SparkExecutionContext.java
index 67efd5c..921219a 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/context/SparkExecutionContext.java
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/context/SparkExecutionContext.java
@@ -224,6 +224,9 @@ public class SparkExecutionContext extends ExecutionContext
_spctx = createContext(conf);
+ if(DMLScript.USE_LOCAL_SPARK_CONFIG)
+ _spctx.setCheckpointDir("/tmp/systemds_spark_cache_" + DMLScript.getUUID());
+
_parRDDs.clear();
}