[systemds] branch master updated: [SYSTEMDS-2632] Fix dynamic recompilation in singlenode exec mode

[mb...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

mboehm7 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/systemds.git


The following commit(s) were added to refs/heads/master by this push:
     new a599a55  [SYSTEMDS-2632] Fix dynamic recompilation in singlenode exec mode
a599a55 is described below

commit a599a55c65c378621979a59c2ccc5f654ab543a0
Author: Matthias Boehm <mb...@gmail.com>
AuthorDate: Sat Aug 22 17:29:36 2020 +0200

    [SYSTEMDS-2632] Fix dynamic recompilation in singlenode exec mode
    
    Our default execution mode is hybrid, but our bin/systemds startup
    script uses singlenode to ensure local operations without the need for
    proper spark configuration. Since the latter entry point is used
    increasingly often, we need to ensure that all default compilation
    techniques nicely work in this context too.
    
    This patch fixes an issue where recompilation of recompile-once
    functions (enabled by IPA) disabled the recompilation of individual
    blocks despite unknown dimensions because the execution type was already
    set to CP (due to the forced singlenode execution).
---
 src/main/java/org/apache/sysds/hops/Hop.java                         | 4 +++-
 .../apache/sysds/runtime/controlprogram/FunctionProgramBlock.java    | 5 ++++-
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/sysds/hops/Hop.java b/src/main/java/org/apache/sysds/hops/Hop.java
index 2558c1d..030bfa8 100644
--- a/src/main/java/org/apache/sysds/hops/Hop.java
+++ b/src/main/java/org/apache/sysds/hops/Hop.java
@@ -200,7 +200,7 @@ public abstract class Hop implements ParseInfo {
 			}
 			else {
 				// enabled with -exec singlenode option
-				_etypeForced = ExecType.CP;  
+				_etypeForced = ExecType.CP;
 			}
 		}
 		else if ( DMLScript.getGlobalExecMode() == ExecMode.SPARK )
@@ -1064,6 +1064,8 @@ public abstract class Hop implements ParseInfo {
 	 * </ul>
 	 */
 	protected void setRequiresRecompileIfNecessary() {
+		//NOTE: when changing these conditions, remember to update the code for
+		//function recompilation in FunctionProgramBlock accordingly
 		boolean caseRemote = (!dimsKnown(true) && _etype == ExecType.SPARK);
 		boolean caseLocal = (!dimsKnown() && _etypeForced == ExecType.CP);
 		boolean caseCodegen = (!dimsKnown() && ConfigurationManager.isCodegenEnabled());
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/FunctionProgramBlock.java b/src/main/java/org/apache/sysds/runtime/controlprogram/FunctionProgramBlock.java
index 5bb183b..cd7f0cc 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/FunctionProgramBlock.java
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/FunctionProgramBlock.java
@@ -25,6 +25,7 @@ import java.util.List;
 import java.util.stream.Collectors;
 
 import org.apache.sysds.api.DMLScript;
+import org.apache.sysds.common.Types.ExecMode;
 import org.apache.sysds.common.Types.FunctionBlock;
 import org.apache.sysds.conf.ConfigurationManager;
 import org.apache.sysds.hops.recompile.Recompiler;
@@ -112,7 +113,9 @@ public class FunctionProgramBlock extends ProgramBlock implements FunctionBlock
 				//     function will be recompiled for every execution.
 				// (2) without reset, there would be no benefit in recompiling the entire function
 				LocalVariableMap tmp = (LocalVariableMap) ec.getVariables().clone();
-				ResetType reset = ConfigurationManager.isCodegenEnabled() ? ResetType.RESET_KNOWN_DIMS : ResetType.RESET;
+				boolean codegen = ConfigurationManager.isCodegenEnabled();
+				boolean singlenode = DMLScript.getGlobalExecMode() == ExecMode.SINGLE_NODE;
+				ResetType reset = (codegen || singlenode) ? ResetType.RESET_KNOWN_DIMS : ResetType.RESET;
 				Recompiler.recompileProgramBlockHierarchy(_childBlocks, tmp, _tid, reset);
 				
 				if( DMLScript.STATISTICS ){