You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemds.apache.org by mb...@apache.org on 2021/09/10 14:34:23 UTC
[systemds] branch master updated: [SYSTEMDS-3127] Fix parfor eval
function loading and registration
This is an automated email from the ASF dual-hosted git repository.
mboehm7 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/systemds.git
The following commit(s) were added to refs/heads/master by this push:
new e959df3 [SYSTEMDS-3127] Fix parfor eval function loading and registration
e959df3 is described below
commit e959df3cd437e78cb9d8003527af375705f370ac
Author: Matthias Boehm <mb...@gmail.com>
AuthorDate: Fri Sep 10 16:34:02 2021 +0200
[SYSTEMDS-3127] Fix parfor eval function loading and registration
This patch fixes issues with complex parfor loops calling a hierarchy of
functions through eval. While the top-level builtin/user functions where
correctly loaded and compiled into thread-specific functions, other
reachable functions were still calling the base versions which might be
incompatible if they have been called separately (not through eval) as
IPA might have propagated literals into these functions. Furthermore,
having thread-specific functions also prevents interference and thread
contention during dynamic recompilation at runtime.
---
scripts/pipelines/scripts/enumerateLogical.dml | 2 +-
.../sysds/runtime/instructions/cp/EvalNaryCPInstruction.java | 2 +-
.../java/org/apache/sysds/runtime/util/ProgramConverter.java | 5 +++++
.../sysds/test/functions/pipelines/BuiltinTopkLogicalTest.java | 9 ++++-----
4 files changed, 11 insertions(+), 7 deletions(-)
diff --git a/scripts/pipelines/scripts/enumerateLogical.dml b/scripts/pipelines/scripts/enumerateLogical.dml
index 1af125e..66b2310 100644
--- a/scripts/pipelines/scripts/enumerateLogical.dml
+++ b/scripts/pipelines/scripts/enumerateLogical.dml
@@ -86,7 +86,7 @@ return (Frame[Unknown] bestLg, Double pre_best)
# # # execute the physical pipelines
scores = matrix(0, nrow(physicalPipList), 1)
- for(i in 1:length(physicalPipList)) {
+ parfor(i in 1:length(physicalPipList)) {
lp2 = as.frame(logicalPipList[as.scalar(ppos[i]),])
pp2 = as.frame(physicalPipList[i,])
# # append configuration keys for extracting the pipeline later on
diff --git a/src/main/java/org/apache/sysds/runtime/instructions/cp/EvalNaryCPInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/cp/EvalNaryCPInstruction.java
index db5676f..1f22e98 100644
--- a/src/main/java/org/apache/sysds/runtime/instructions/cp/EvalNaryCPInstruction.java
+++ b/src/main/java/org/apache/sysds/runtime/instructions/cp/EvalNaryCPInstruction.java
@@ -113,7 +113,7 @@ public class EvalNaryCPInstruction extends BuiltinNaryCPInstruction {
if( ProgramBlock.isThreadID(_threadID) && ParForProgramBlock.COPY_EVAL_FUNCTIONS ) {
String funcNameParfor = funcName + Lop.CP_CHILD_THREAD + _threadID;
if( !ec.getProgram().containsFunctionProgramBlock(nsName, funcNameParfor, false) ) { //copy on demand
- fpb = ProgramConverter.createDeepCopyFunctionProgramBlock(fpb, new HashSet<>(), new HashSet<>());
+ fpb = ProgramConverter.createDeepCopyFunctionProgramBlock(fpb, new HashSet<>(), new HashSet<>(), _threadID);
ec.getProgram().addFunctionProgramBlock(nsName, funcNameParfor, fpb, false);
}
fpb = ec.getProgram().getFunctionProgramBlock(nsName, funcNameParfor, false);
diff --git a/src/main/java/org/apache/sysds/runtime/util/ProgramConverter.java b/src/main/java/org/apache/sysds/runtime/util/ProgramConverter.java
index 18a88fc..c892ffb 100644
--- a/src/main/java/org/apache/sysds/runtime/util/ProgramConverter.java
+++ b/src/main/java/org/apache/sysds/runtime/util/ProgramConverter.java
@@ -410,6 +410,11 @@ public class ProgramConverter
return createDeepCopyFunctionProgramBlock(fpb, fnStack, fnCreated, 0, -1, true);
}
+ public static FunctionProgramBlock createDeepCopyFunctionProgramBlock(FunctionProgramBlock fpb, Set<String> fnStack, Set<String> fnCreated, long pid) {
+ //recursive deep copy with creation of thread-specific function calls
+ return createDeepCopyFunctionProgramBlock(fpb, fnStack, fnCreated, pid, -1, false);
+ }
+
public static FunctionProgramBlock createDeepCopyFunctionProgramBlock(FunctionProgramBlock fpb, Set<String> fnStack, Set<String> fnCreated, long pid, int IDPrefix, boolean plain)
{
if( fpb == null )
diff --git a/src/test/java/org/apache/sysds/test/functions/pipelines/BuiltinTopkLogicalTest.java b/src/test/java/org/apache/sysds/test/functions/pipelines/BuiltinTopkLogicalTest.java
index 35d9843..5818128 100644
--- a/src/test/java/org/apache/sysds/test/functions/pipelines/BuiltinTopkLogicalTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/pipelines/BuiltinTopkLogicalTest.java
@@ -20,6 +20,7 @@
package org.apache.sysds.test.functions.pipelines;
import org.apache.sysds.common.Types;
+import org.apache.sysds.common.Types.ExecMode;
import org.apache.sysds.test.AutomatedTestBase;
import org.apache.sysds.test.TestConfiguration;
import org.apache.sysds.test.TestUtils;
@@ -49,19 +50,17 @@ public class BuiltinTopkLogicalTest extends AutomatedTestBase {
@Test
public void testLogical1() {
- runTestLogical(10, 3, 2, Types.ExecMode.SINGLE_NODE);
+ runTestLogical(10, 5, 2, ExecMode.SINGLE_NODE);
}
@Test
public void testLogical2() {
- runTestLogical(2, 2, 2,
- Types.ExecMode.SINGLE_NODE);
+ runTestLogical(2, 2, 2, ExecMode.SINGLE_NODE);
}
@Test
public void testLogicalHybrid() {
- runTestLogical(3, 3, 2,
- Types.ExecMode.HYBRID);
+ runTestLogical(3, 3, 2, ExecMode.HYBRID);
}
private void runTestLogical(int max_iter, int num_inst, int num_exec, Types.ExecMode et) {