You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemds.apache.org by mb...@apache.org on 2025/04/06 08:27:41 UTC
(systemds) branch main updated: [SYSTEMDS-3846] Perftest: Fix parfor forced CP compilation
This is an automated email from the ASF dual-hosted git repository.
mboehm7 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/systemds.git
The following commit(s) were added to refs/heads/main by this push:
new 4d6c6b4cb7 [SYSTEMDS-3846] Perftest: Fix parfor forced CP compilation
4d6c6b4cb7 is described below
commit 4d6c6b4cb71c4ea1dc818739e601ab6a735c55ac
Author: Matthias Boehm <mb...@gmail.com>
AuthorDate: Sun Apr 6 10:20:24 2025 +0200
[SYSTEMDS-3846] Perftest: Fix parfor forced CP compilation
The perftest runMSVM_10k_1k_dense_k5 reproducible failed on serializing
the parfor body program for a remote spark parfor job, because there
were remaining spark instructions (checkpoints). The reason was because
the forced CP compilation before such remote jobs was only applied in
a subset of cases and special hop properties were not correctly cleaned
up. This patch fixes the general issue.
---
src/main/java/org/apache/sysds/hops/recompile/Recompiler.java | 6 +++++-
.../org/apache/sysds/runtime/controlprogram/ParForProgramBlock.java | 3 +--
2 files changed, 6 insertions(+), 3 deletions(-)
diff --git a/src/main/java/org/apache/sysds/hops/recompile/Recompiler.java b/src/main/java/org/apache/sysds/hops/recompile/Recompiler.java
index 63d0f1eab6..0aef8a1583 100644
--- a/src/main/java/org/apache/sysds/hops/recompile/Recompiler.java
+++ b/src/main/java/org/apache/sysds/hops/recompile/Recompiler.java
@@ -1509,8 +1509,12 @@ public class Recompiler {
public static void rSetExecType( Hop hop, ExecType etype ) {
if( hop.isVisited() )
return;
- //update function names
+ //update hop properties (exec type, spark lops)
hop.setForcedExecType(etype);
+ if(etype == ExecType.CP) {
+ hop.setRequiresCheckpoint(false);
+ hop.setRequiresReblock(false);
+ }
if( hop.getInput() != null )
for( Hop c : hop.getInput() )
rSetExecType(c, etype);
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/ParForProgramBlock.java b/src/main/java/org/apache/sysds/runtime/controlprogram/ParForProgramBlock.java
index 4c492131d9..85f41ad7fe 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/ParForProgramBlock.java
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/ParForProgramBlock.java
@@ -834,8 +834,7 @@ public class ParForProgramBlock extends ForProgramBlock {
{
// Step 0) check and compile to CP (if forced remote parfor)
boolean flagForced = false;
- if( FORCE_CP_ON_REMOTE_SPARK && (_optMode == POptMode.NONE
- || (_optMode == POptMode.CONSTRAINED && _execMode==PExecMode.REMOTE_SPARK)) ) {
+ if( FORCE_CP_ON_REMOTE_SPARK && _execMode==PExecMode.REMOTE_SPARK ) {
//tid = 0 because replaced in remote parworker
flagForced = checkSparkAndRecompileToCP(0);
}