You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2020/06/11 13:57:44 UTC
[systemml] branch master updated: [SYSTEMDS-412] Fix lineage-based
reuse for update-inplace indexing
This is an automated email from the ASF dual-hosted git repository.
mboehm7 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/systemml.git
The following commit(s) were added to refs/heads/master by this push:
new 3d876d3 [SYSTEMDS-412] Fix lineage-based reuse for update-inplace indexing
3d876d3 is described below
commit 3d876d33ad019fe026799daaaaf540fc20a86ec4
Author: Matthias Boehm <mb...@gmail.com>
AuthorDate: Thu Jun 11 15:57:20 2020 +0200
[SYSTEMDS-412] Fix lineage-based reuse for update-inplace indexing
This patch disabled lineage-based reuse for update-inplace left indexing
operations as reuse would create incorrect results due to later in-place
updates the change the cached data object.
Furthermore, this patch also aims to make the codegen tests for robust
wrt the surefire github action integration (less explain output).
---
.../runtime/controlprogram/context/SparkExecutionContext.java | 8 ++------
.../java/org/apache/sysds/runtime/lineage/LineageCacheConfig.java | 5 ++++-
.../org/apache/sysds/test/functions/codegen/CellwiseTmplTest.java | 2 +-
.../apache/sysds/test/functions/codegen/DAGCellwiseTmplTest.java | 2 +-
.../org/apache/sysds/test/functions/codegen/MiscPatternTest.java | 2 +-
.../org/apache/sysds/test/functions/codegen/MultiAggTmplTest.java | 2 +-
.../apache/sysds/test/functions/codegen/OuterProdTmplTest.java | 2 +-
.../org/apache/sysds/test/functions/codegen/RowAggTmplTest.java | 2 +-
.../sysds/test/functions/codegen/RowConv2DOperationsTest.java | 2 +-
.../sysds/test/functions/codegen/RowVectorComparisonTest.java | 2 +-
.../apache/sysds/test/functions/codegen/SparseSideInputTest.java | 2 +-
.../apache/sysds/test/functions/codegen/SumProductChainTest.java | 2 +-
12 files changed, 16 insertions(+), 17 deletions(-)
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/context/SparkExecutionContext.java b/src/main/java/org/apache/sysds/runtime/controlprogram/context/SparkExecutionContext.java
index a1e2b92..11a4e93 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/context/SparkExecutionContext.java
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/context/SparkExecutionContext.java
@@ -171,18 +171,14 @@ public class SparkExecutionContext extends ExecutionContext
_spctx = null;
}
- public void close()
- {
+ public void close() {
synchronized( SparkExecutionContext.class ) {
- if( _spctx != null )
- {
+ if( _spctx != null ) {
//stop the spark context if existing
_spctx.stop();
-
//make sure stopped context is never used again
_spctx = null;
}
-
}
}
diff --git a/src/main/java/org/apache/sysds/runtime/lineage/LineageCacheConfig.java b/src/main/java/org/apache/sysds/runtime/lineage/LineageCacheConfig.java
index 48a512a..22964ba 100644
--- a/src/main/java/org/apache/sysds/runtime/lineage/LineageCacheConfig.java
+++ b/src/main/java/org/apache/sysds/runtime/lineage/LineageCacheConfig.java
@@ -25,6 +25,7 @@ import org.apache.sysds.runtime.controlprogram.context.ExecutionContext;
import org.apache.sysds.runtime.instructions.Instruction;
import org.apache.sysds.runtime.instructions.cp.ComputationCPInstruction;
import org.apache.sysds.runtime.instructions.cp.ListIndexingCPInstruction;
+import org.apache.sysds.runtime.instructions.cp.MatrixIndexingCPInstruction;
import java.util.Comparator;
@@ -151,7 +152,9 @@ public class LineageCacheConfig
&& !(inst instanceof ListIndexingCPInstruction);
boolean rightop = (ArrayUtils.contains(REUSE_OPCODES, inst.getOpcode())
|| (inst.getOpcode().equals("append") && isVectorAppend(inst, ec)));
- return insttype && rightop;
+ boolean updateInplace = (inst instanceof MatrixIndexingCPInstruction)
+ && ec.getMatrixObject(((ComputationCPInstruction)inst).input1).getUpdateType().isInPlace();
+ return insttype && rightop && !updateInplace;
}
private static boolean isVectorAppend(Instruction inst, ExecutionContext ec) {
diff --git a/src/test/java/org/apache/sysds/test/functions/codegen/CellwiseTmplTest.java b/src/test/java/org/apache/sysds/test/functions/codegen/CellwiseTmplTest.java
index bd369c0..b1d184c 100644
--- a/src/test/java/org/apache/sysds/test/functions/codegen/CellwiseTmplTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/codegen/CellwiseTmplTest.java
@@ -477,7 +477,7 @@ public class CellwiseTmplTest extends AutomatedTestBase
String HOME = SCRIPT_DIR + TEST_DIR;
fullDMLScriptName = HOME + testname + ".dml";
- programArgs = new String[]{"-explain", "-stats", "-args", output("S") };
+ programArgs = new String[]{"-stats", "-args", output("S") };
fullRScriptName = HOME + testname + ".R";
rCmd = getRCmd(inputDir(), expectedDir());
diff --git a/src/test/java/org/apache/sysds/test/functions/codegen/DAGCellwiseTmplTest.java b/src/test/java/org/apache/sysds/test/functions/codegen/DAGCellwiseTmplTest.java
index c48db84..457285c 100644
--- a/src/test/java/org/apache/sysds/test/functions/codegen/DAGCellwiseTmplTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/codegen/DAGCellwiseTmplTest.java
@@ -127,7 +127,7 @@ public class DAGCellwiseTmplTest extends AutomatedTestBase
String HOME = SCRIPT_DIR + TEST_DIR;
fullDMLScriptName = HOME + testname + ".dml";
- programArgs = new String[]{"-explain", "hops", "-stats",
+ programArgs = new String[]{"hops", "-stats",
"-args", String.valueOf(cols), output("S") };
fullRScriptName = HOME + testname + ".R";
diff --git a/src/test/java/org/apache/sysds/test/functions/codegen/MiscPatternTest.java b/src/test/java/org/apache/sysds/test/functions/codegen/MiscPatternTest.java
index 60309b4..08c631c 100644
--- a/src/test/java/org/apache/sysds/test/functions/codegen/MiscPatternTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/codegen/MiscPatternTest.java
@@ -126,7 +126,7 @@ public class MiscPatternTest extends AutomatedTestBase
String HOME = SCRIPT_DIR + TEST_DIR;
fullDMLScriptName = HOME + testname + ".dml";
- programArgs = new String[]{"-explain", "recompile_runtime", "-stats", "-args", output("S") };
+ programArgs = new String[]{"recompile_runtime", "-stats", "-args", output("S") };
fullRScriptName = HOME + testname + ".R";
rCmd = getRCmd(expectedDir());
diff --git a/src/test/java/org/apache/sysds/test/functions/codegen/MultiAggTmplTest.java b/src/test/java/org/apache/sysds/test/functions/codegen/MultiAggTmplTest.java
index d452313..d0b1c5a 100644
--- a/src/test/java/org/apache/sysds/test/functions/codegen/MultiAggTmplTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/codegen/MultiAggTmplTest.java
@@ -174,7 +174,7 @@ public class MultiAggTmplTest extends AutomatedTestBase
String HOME = SCRIPT_DIR + TEST_DIR;
fullDMLScriptName = HOME + testname + ".dml";
- programArgs = new String[]{"-explain", "-stats", "-args", output("S") };
+ programArgs = new String[]{"-stats", "-args", output("S") };
fullRScriptName = HOME + testname + ".R";
rCmd = getRCmd(inputDir(), expectedDir());
diff --git a/src/test/java/org/apache/sysds/test/functions/codegen/OuterProdTmplTest.java b/src/test/java/org/apache/sysds/test/functions/codegen/OuterProdTmplTest.java
index b4cdc96..d64f7ed 100644
--- a/src/test/java/org/apache/sysds/test/functions/codegen/OuterProdTmplTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/codegen/OuterProdTmplTest.java
@@ -265,7 +265,7 @@ public class OuterProdTmplTest extends AutomatedTestBase
String HOME = SCRIPT_DIR + TEST_DIR;
fullDMLScriptName = HOME + testname + ".dml";
- programArgs = new String[]{"-explain", "-stats", "-args", output("S"), input("A")};
+ programArgs = new String[]{"-stats", "-args", output("S"), input("A")};
fullRScriptName = HOME + testname + ".R";
rCmd = getRCmd(inputDir(), expectedDir());
diff --git a/src/test/java/org/apache/sysds/test/functions/codegen/RowAggTmplTest.java b/src/test/java/org/apache/sysds/test/functions/codegen/RowAggTmplTest.java
index 10a816c..695ee51 100644
--- a/src/test/java/org/apache/sysds/test/functions/codegen/RowAggTmplTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/codegen/RowAggTmplTest.java
@@ -799,7 +799,7 @@ public class RowAggTmplTest extends AutomatedTestBase
String HOME = SCRIPT_DIR + TEST_DIR;
fullDMLScriptName = HOME + testname + ".dml";
- programArgs = new String[]{"-explain", "-stats", "-args", output("S") };
+ programArgs = new String[]{"-stats", "-args", output("S") };
fullRScriptName = HOME + testname + ".R";
rCmd = getRCmd(inputDir(), expectedDir());
diff --git a/src/test/java/org/apache/sysds/test/functions/codegen/RowConv2DOperationsTest.java b/src/test/java/org/apache/sysds/test/functions/codegen/RowConv2DOperationsTest.java
index e76b1ab..c80508d 100644
--- a/src/test/java/org/apache/sysds/test/functions/codegen/RowConv2DOperationsTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/codegen/RowConv2DOperationsTest.java
@@ -84,7 +84,7 @@ public class RowConv2DOperationsTest extends AutomatedTestBase
String HOME = SCRIPT_DIR + TEST_DIR;
fullDMLScriptName = HOME + testname + ".dml";
- programArgs = new String[]{"-explain", "recompile_runtime", "-stats", "-args",
+ programArgs = new String[]{"recompile_runtime", "-stats", "-args",
String.valueOf(imgSize), String.valueOf(numImg), String.valueOf(numChannels),
String.valueOf(numFilters), String.valueOf(filterSize), String.valueOf(stride),
String.valueOf(pad), output("B"), sparseVal1, sparseVal2 };
diff --git a/src/test/java/org/apache/sysds/test/functions/codegen/RowVectorComparisonTest.java b/src/test/java/org/apache/sysds/test/functions/codegen/RowVectorComparisonTest.java
index cbbc9c3..c33a471 100644
--- a/src/test/java/org/apache/sysds/test/functions/codegen/RowVectorComparisonTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/codegen/RowVectorComparisonTest.java
@@ -132,7 +132,7 @@ public class RowVectorComparisonTest extends AutomatedTestBase
String HOME = SCRIPT_DIR + TEST_DIR;
fullDMLScriptName = HOME + testname + ".dml";
- programArgs = new String[]{"-explain", "-stats", "-args", output("R") };
+ programArgs = new String[]{"-stats", "-args", output("R") };
fullRScriptName = HOME + testname + ".R";
rCmd = getRCmd(inputDir(), expectedDir());
diff --git a/src/test/java/org/apache/sysds/test/functions/codegen/SparseSideInputTest.java b/src/test/java/org/apache/sysds/test/functions/codegen/SparseSideInputTest.java
index b9a2bb7..74460e4 100644
--- a/src/test/java/org/apache/sysds/test/functions/codegen/SparseSideInputTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/codegen/SparseSideInputTest.java
@@ -153,7 +153,7 @@ public class SparseSideInputTest extends AutomatedTestBase
String HOME = SCRIPT_DIR + TEST_DIR;
fullDMLScriptName = HOME + testname + ".dml";
- programArgs = new String[]{"-explain", "-stats", "-args",
+ programArgs = new String[]{"-stats", "-args",
input("X"), input("Y"), output("R") };
fullRScriptName = HOME + testname + ".R";
diff --git a/src/test/java/org/apache/sysds/test/functions/codegen/SumProductChainTest.java b/src/test/java/org/apache/sysds/test/functions/codegen/SumProductChainTest.java
index 663dcad..f674f88 100644
--- a/src/test/java/org/apache/sysds/test/functions/codegen/SumProductChainTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/codegen/SumProductChainTest.java
@@ -110,7 +110,7 @@ public class SumProductChainTest extends AutomatedTestBase
String HOME = SCRIPT_DIR + TEST_DIR;
fullDMLScriptName = HOME + testname + ".dml";
- programArgs = new String[]{"-explain", "hops", "-stats",
+ programArgs = new String[]{"hops", "-stats",
"-args", input("X"), output("R") };
fullRScriptName = HOME + testname + ".R";