You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemds.apache.org by ar...@apache.org on 2021/06/02 16:18:30 UTC
[systemds] branch master updated: [SYSTEMDS-2889] Add initial
lineage based debugger
This is an automated email from the ASF dual-hosted git repository.
arnabp20 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/systemds.git
The following commit(s) were added to refs/heads/master by this push:
new fd4b0d0 [SYSTEMDS-2889] Add initial lineage based debugger
fd4b0d0 is described below
commit fd4b0d05a38a29567c2a44d6c872c1f7d120cef8
Author: Benjamin Rath <b....@trever.io>
AuthorDate: Wed Jun 2 18:18:07 2021 +0200
[SYSTEMDS-2889] Add initial lineage based debugger
This patch adds a lineage based debugger which helps tracking
the first operator responsible for a constraint violation.
Currently, the supported constraints are NAN, positive infinity
and negative infinity.
AMLS project SS2021.
Closes #1286.
---
src/main/java/org/apache/sysds/api/DMLOptions.java | 3 +
src/main/java/org/apache/sysds/api/DMLScript.java | 2 +
.../controlprogram/context/ExecutionContext.java | 10 +-
.../runtime/instructions/cp/CPInstruction.java | 3 +
.../runtime/instructions/gpu/GPUInstruction.java | 3 +
.../sysds/runtime/lineage/BooleanArray32.java | 45 ++++
.../sysds/runtime/lineage/LineageDebugger.java | 108 ++++++++++
.../apache/sysds/runtime/lineage/LineageItem.java | 36 +++-
.../sysds/runtime/lineage/LineageItemUtils.java | 3 +
.../sysds/runtime/lineage/LineageParser.java | 6 +-
src/test/java/org/apache/sysds/test/TestUtils.java | 4 +
.../functions/lineage/LineageDebuggerTest.java | 236 +++++++++++++++++++++
.../scripts/functions/lineage/LineageDebugger1.dml | 39 ++++
13 files changed, 487 insertions(+), 11 deletions(-)
diff --git a/src/main/java/org/apache/sysds/api/DMLOptions.java b/src/main/java/org/apache/sysds/api/DMLOptions.java
index a4dae4b..0949c06 100644
--- a/src/main/java/org/apache/sysds/api/DMLOptions.java
+++ b/src/main/java/org/apache/sysds/api/DMLOptions.java
@@ -66,6 +66,7 @@ public class DMLOptions {
public ReuseCacheType linReuseType = ReuseCacheType.NONE; // reuse type (full, partial, hybrid)
public LineageCachePolicy linCachePolicy= LineageCachePolicy.COSTNSIZE; // lineage cache eviction policy
public boolean lineage_estimate = false; // whether estimate reuse benefits
+ public boolean lineage_debugger = false; // whether enable lineage debugger
public boolean fedWorker = false;
public int fedWorkerPort = -1;
public boolean checkPrivacy = false; // Check which privacy constraints are loaded and checked during federated execution
@@ -140,6 +141,8 @@ public class DMLOptions {
dmlOptions.linCachePolicy = LineageCachePolicy.DAGHEIGHT;
else if (lineageType.equalsIgnoreCase("estimate"))
dmlOptions.lineage_estimate = lineageType.equalsIgnoreCase("estimate");
+ else if (lineageType.equalsIgnoreCase("debugger"))
+ dmlOptions.lineage_debugger = lineageType.equalsIgnoreCase("debugger");
else
throw new org.apache.commons.cli.ParseException(
"Invalid argument specified for -lineage option: " + lineageType);
diff --git a/src/main/java/org/apache/sysds/api/DMLScript.java b/src/main/java/org/apache/sysds/api/DMLScript.java
index c60fa3e..5072830 100644
--- a/src/main/java/org/apache/sysds/api/DMLScript.java
+++ b/src/main/java/org/apache/sysds/api/DMLScript.java
@@ -100,6 +100,7 @@ public class DMLScript
public static ReuseCacheType LINEAGE_REUSE = DMLOptions.defaultOptions.linReuseType; // whether lineage-based reuse
public static LineageCachePolicy LINEAGE_POLICY = DMLOptions.defaultOptions.linCachePolicy; // lineage cache eviction policy
public static boolean LINEAGE_ESTIMATE = DMLOptions.defaultOptions.lineage_estimate; // whether estimate reuse benefits
+ public static boolean LINEAGE_DEBUGGER = DMLOptions.defaultOptions.lineage_debugger; // whether enable lineage debugger
public static boolean CHECK_PRIVACY = DMLOptions.defaultOptions.checkPrivacy; // Check which privacy constraints are loaded and checked during federated execution
public static boolean USE_ACCELERATOR = DMLOptions.defaultOptions.gpu;
@@ -224,6 +225,7 @@ public class DMLScript
LINEAGE_POLICY = dmlOptions.linCachePolicy;
LINEAGE_ESTIMATE = dmlOptions.lineage_estimate;
CHECK_PRIVACY = dmlOptions.checkPrivacy;
+ LINEAGE_DEBUGGER = dmlOptions.lineage_debugger;
String fnameOptConfig = dmlOptions.configFile;
boolean isFile = dmlOptions.filePath != null;
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/context/ExecutionContext.java b/src/main/java/org/apache/sysds/runtime/controlprogram/context/ExecutionContext.java
index f0448f1..6b1054b 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/context/ExecutionContext.java
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/context/ExecutionContext.java
@@ -48,6 +48,7 @@ import org.apache.sysds.runtime.instructions.gpu.context.CSRPointer;
import org.apache.sysds.runtime.instructions.gpu.context.GPUContext;
import org.apache.sysds.runtime.instructions.gpu.context.GPUObject;
import org.apache.sysds.runtime.lineage.Lineage;
+import org.apache.sysds.runtime.lineage.LineageDebugger;
import org.apache.sysds.runtime.lineage.LineageItem;
import org.apache.sysds.runtime.matrix.data.FrameBlock;
import org.apache.sysds.runtime.matrix.data.MatrixBlock;
@@ -808,9 +809,16 @@ public class ExecutionContext {
public void traceLineage(Instruction inst) {
if( _lineage == null )
throw new DMLRuntimeException("Lineage Trace unavailable.");
+ // TODO bra: store all newly created lis in active list
_lineage.trace(inst, this);
}
-
+
+ public void maintainLineageDebuggerInfo(Instruction inst) {
+ if( _lineage == null )
+ throw new DMLRuntimeException("Lineage Trace unavailable.");
+ LineageDebugger.maintainSpecialValueBits(_lineage, inst, this);
+ }
+
public LineageItem getLineageItem(CPOperand input) {
if( _lineage == null )
throw new DMLRuntimeException("Lineage Trace unavailable.");
diff --git a/src/main/java/org/apache/sysds/runtime/instructions/cp/CPInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/cp/CPInstruction.java
index 014b854..84b2332 100644
--- a/src/main/java/org/apache/sysds/runtime/instructions/cp/CPInstruction.java
+++ b/src/main/java/org/apache/sysds/runtime/instructions/cp/CPInstruction.java
@@ -152,6 +152,9 @@ public abstract class CPInstruction extends Instruction
if (!LineageCacheConfig.ReuseCacheType.isNone() && DMLScript.USE_ACCELERATOR
&& LineageCacheConfig.CONCURRENTGPUEVICTION)
LineageCacheConfig.STOPBACKGROUNDEVICTION = true;
+
+ if (DMLScript.LINEAGE_DEBUGGER)
+ ec.maintainLineageDebuggerInfo(this);
}
/**
diff --git a/src/main/java/org/apache/sysds/runtime/instructions/gpu/GPUInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/gpu/GPUInstruction.java
index 4c51c70..700145b 100644
--- a/src/main/java/org/apache/sysds/runtime/instructions/gpu/GPUInstruction.java
+++ b/src/main/java/org/apache/sysds/runtime/instructions/gpu/GPUInstruction.java
@@ -226,6 +226,9 @@ public abstract class GPUInstruction extends Instruction implements LineageTrace
if(gpuCtx != null)
gpuCtx.printMemoryInfo(getOpcode());
}
+
+ //default post-process behavior
+ super.postprocessInstruction(ec);
}
/**
diff --git a/src/main/java/org/apache/sysds/runtime/lineage/BooleanArray32.java b/src/main/java/org/apache/sysds/runtime/lineage/BooleanArray32.java
new file mode 100644
index 0000000..f3cc5e6
--- /dev/null
+++ b/src/main/java/org/apache/sysds/runtime/lineage/BooleanArray32.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysds.runtime.lineage;
+
+public class BooleanArray32 {
+ private int _value;
+
+ public BooleanArray32(int value){
+ _value = value;
+ }
+
+ public boolean get(int pos) {
+ return (_value & (1 << pos)) != 0;
+ }
+
+ public void set(int pos, boolean value) {
+ int mask = 1 << pos;
+ _value = (_value & ~mask) | (value ? mask : 0);
+ }
+
+ public int getValue() {
+ return _value;
+ }
+
+ public void setValue(int value) {
+ _value = value;
+ }
+}
diff --git a/src/main/java/org/apache/sysds/runtime/lineage/LineageDebugger.java b/src/main/java/org/apache/sysds/runtime/lineage/LineageDebugger.java
new file mode 100644
index 0000000..1730c5a
--- /dev/null
+++ b/src/main/java/org/apache/sysds/runtime/lineage/LineageDebugger.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysds.runtime.lineage;
+
+
+import org.apache.sysds.runtime.controlprogram.caching.CacheableData;
+import org.apache.sysds.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysds.runtime.instructions.Instruction;
+import org.apache.sysds.runtime.instructions.cp.*;
+import org.apache.sysds.runtime.matrix.data.MatrixBlock;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Queue;
+import java.util.LinkedList;
+
+public class LineageDebugger {
+ public static final int POS_NAN = 0;
+ public static final int POS_POSITIVE_INFINITY = 1;
+ public static final int POS_NEGATIVE_INFINITY = 2;
+
+ public static void maintainSpecialValueBits(Lineage lineage, Instruction inst, ExecutionContext ec) {
+ ArrayList<CPOperand> outputs = new ArrayList<>();
+
+ // Only CP instructions are supported right now
+ CPOperand singleOutput =
+ inst instanceof ComputationCPInstruction ? ((ComputationCPInstruction) inst).getOutput() :
+ inst instanceof BuiltinNaryCPInstruction ? ((BuiltinNaryCPInstruction) inst).getOutput() :
+ inst instanceof SqlCPInstruction ? ((SqlCPInstruction) inst).getOutput() :
+ inst instanceof VariableCPInstruction ? ((VariableCPInstruction) inst).getOutput() :
+ null;
+ if (singleOutput != null)
+ outputs.add(singleOutput);
+
+ Collection<CPOperand> multiOutputs =
+ inst instanceof MultiReturnBuiltinCPInstruction ? ((MultiReturnBuiltinCPInstruction) inst).getOutputs() :
+ inst instanceof MultiReturnParameterizedBuiltinCPInstruction ? ((MultiReturnParameterizedBuiltinCPInstruction) inst).getOutputs() :
+ null;
+ if (multiOutputs != null)
+ outputs.addAll(multiOutputs);
+
+ for (CPOperand output : outputs) {
+ CacheableData<MatrixBlock> cd = ec.getMatrixObject(output);
+ MatrixBlock mb = cd.acquireReadAndRelease();
+ LineageItem li = lineage.get(output);
+
+ updateSpecialValueBit(mb, li, POS_NAN, Double.NaN);
+ updateSpecialValueBit(mb, li, POS_POSITIVE_INFINITY, Double.POSITIVE_INFINITY);
+ updateSpecialValueBit(mb, li, POS_NEGATIVE_INFINITY, Double.NEGATIVE_INFINITY);
+ }
+ }
+
+ public static LineageItem firstOccurrenceOfNR(LineageItem li, int pos) {
+ if (!li.getSpecialValueBit(pos))
+ return null;
+
+ LineageItem tmp;
+ Queue<LineageItem> q = new LinkedList<>();
+ q.add(li);
+
+ while ((tmp = q.poll()) != null) {
+ if (tmp.isVisited())
+ continue;
+
+ if (tmp.getInputs() != null) {
+ boolean flag = false;
+ for (LineageItem in : tmp.getInputs()) {
+ flag |= in.getSpecialValueBit(pos);
+ q.add(in);
+ }
+ if (!flag)
+ break;
+ }
+ tmp.setVisited(true);
+ }
+ li.resetVisitStatusNR();
+ return tmp;
+ }
+
+ private static void updateSpecialValueBit(MatrixBlock mb, LineageItem li, int pos, double pattern) {
+ boolean flag = false;
+ for (LineageItem input : li.getInputs()) {
+ if (input.getSpecialValueBit(pos)) {
+ flag = true;
+ break;
+ }
+ }
+ flag |= mb.containsValue(pattern);
+ li.setSpecialValueBit(pos, flag);
+ }
+}
diff --git a/src/main/java/org/apache/sysds/runtime/lineage/LineageItem.java b/src/main/java/org/apache/sysds/runtime/lineage/LineageItem.java
index cd2346d..14b9894 100644
--- a/src/main/java/org/apache/sysds/runtime/lineage/LineageItem.java
+++ b/src/main/java/org/apache/sysds/runtime/lineage/LineageItem.java
@@ -37,6 +37,7 @@ public class LineageItem {
private int _hash = 0;
private LineageItem _dedupPatch;
private long _distLeaf2Node;
+ private final BooleanArray32 _specialValueBits; // TODO: Move this to a new subclass
// init visited to true to ensure visited items are
// not hidden when used as inputs to new items
private boolean _visited = true;
@@ -53,30 +54,30 @@ public class LineageItem {
}
public LineageItem(long id, String data) {
- this(id, data, "", null);
+ this(id, data, "", null, 0);
}
public LineageItem(String data, String opcode) {
- this(_idSeq.getNextID(), data, opcode, null);
+ this(_idSeq.getNextID(), data, opcode, null, 0);
}
public LineageItem(String opcode, LineageItem[] inputs) {
- this(_idSeq.getNextID(), "", opcode, inputs);
+ this(_idSeq.getNextID(), "", opcode, inputs, 0);
}
public LineageItem(String data, String opcode, LineageItem[] inputs) {
- this(_idSeq.getNextID(), data, opcode, inputs);
+ this(_idSeq.getNextID(), data, opcode, inputs, 0);
}
public LineageItem(String opcode, LineageItem dedupPatch, LineageItem[] inputs) {
- this(_idSeq.getNextID(), "", opcode, inputs);
+ this(_idSeq.getNextID(), "", opcode, inputs, 0);
// maintain a pointer to the dedup patch
_dedupPatch = dedupPatch;
_hash = _dedupPatch._hash;
}
public LineageItem(String opcode, LineageItem dedupPatch, int dpatchHash, LineageItem[] inputs) {
- this(_idSeq.getNextID(), "", opcode, inputs);
+ this(_idSeq.getNextID(), "", opcode, inputs, 0);
// maintain a pointer to the dedup patch
_dedupPatch = dedupPatch;
_hash = dpatchHash;
@@ -87,14 +88,14 @@ public class LineageItem {
}
public LineageItem(long id, LineageItem li) {
- this(id, li._data, li._opcode, li._inputs);
+ this(id, li._data, li._opcode, li._inputs, 0);
}
public LineageItem(long id, String data, String opcode) {
- this(id, data, opcode, null);
+ this(id, data, opcode, null, 0);
}
- public LineageItem(long id, String data, String opcode, LineageItem[] inputs) {
+ public LineageItem(long id, String data, String opcode, LineageItem[] inputs, int specialValueBits) {
_id = id;
_opcode = opcode;
_data = data;
@@ -104,6 +105,7 @@ public class LineageItem {
_hash = hashCode();
// store the distance of this node from the leaves. (O(#inputs)) operation
_distLeaf2Node = distLeaf2Node();
+ _specialValueBits = new BooleanArray32(specialValueBits);
}
public LineageItem[] getInputs() {
@@ -142,6 +144,14 @@ public class LineageItem {
_visited = flag;
}
+ public void setSpecialValueBit(int pos, boolean flag) {
+ _specialValueBits.set(pos, flag);
+ }
+
+ public void setSpecialValueBits(int value) {
+ _specialValueBits.setValue(value);
+ }
+
private long distLeaf2Node() {
// Derive height only if the corresponding reuse
// policy is selected, otherwise set -1.
@@ -169,6 +179,14 @@ public class LineageItem {
return _opcode;
}
+ public boolean getSpecialValueBit(int pos) {
+ return _specialValueBits.get(pos);
+ }
+
+ public int getSpecialValueBits() {
+ return _specialValueBits.getValue();
+ }
+
public boolean isPlaceholder() {
return _opcode.startsWith(LineageItemUtils.LPLACEHOLDER);
}
diff --git a/src/main/java/org/apache/sysds/runtime/lineage/LineageItemUtils.java b/src/main/java/org/apache/sysds/runtime/lineage/LineageItemUtils.java
index edd9a73..ca8d2cb 100644
--- a/src/main/java/org/apache/sysds/runtime/lineage/LineageItemUtils.java
+++ b/src/main/java/org/apache/sysds/runtime/lineage/LineageItemUtils.java
@@ -135,6 +135,9 @@ public class LineageItemUtils {
.map(i -> String.format("(%d)", i.getId()))
.collect(Collectors.joining(" "));
sb.append(ids);
+
+ if (DMLScript.LINEAGE_DEBUGGER)
+ sb.append(" ").append("[").append(li.getSpecialValueBits()).append("]");
}
return sb.toString().trim();
}
diff --git a/src/main/java/org/apache/sysds/runtime/lineage/LineageParser.java b/src/main/java/org/apache/sysds/runtime/lineage/LineageParser.java
index 46dff8e..439fe1f 100644
--- a/src/main/java/org/apache/sysds/runtime/lineage/LineageParser.java
+++ b/src/main/java/org/apache/sysds/runtime/lineage/LineageParser.java
@@ -108,15 +108,19 @@ public class LineageParser
}*/
ArrayList<LineageItem> inputs = new ArrayList<>();
+ int specialValueBits = 0;
for( int i=1; i<tokens.length; i++ ) {
String token = tokens[i];
if (token.startsWith("(") && token.endsWith(")")) {
token = token.substring(1, token.length()-1); //rm parentheses
inputs.add(map.get(Long.valueOf(token)));
+ } else if (token.startsWith("[") && token.endsWith("]")) {
+ token = token.substring(1, token.length() - 1); //rm parentheses
+ specialValueBits = Integer.parseInt(token);
} else
throw new ParseException("Invalid format for LineageItem reference");
}
- return new LineageItem(id, "", opcode, inputs.toArray(new LineageItem[0]));
+ return new LineageItem(id, "", opcode, inputs.toArray(new LineageItem[0]), specialValueBits);
}
protected static void parseLineageTraceDedup(String str) {
diff --git a/src/test/java/org/apache/sysds/test/TestUtils.java b/src/test/java/org/apache/sysds/test/TestUtils.java
index 42cec58..e8fdb2f 100644
--- a/src/test/java/org/apache/sysds/test/TestUtils.java
+++ b/src/test/java/org/apache/sysds/test/TestUtils.java
@@ -1020,6 +1020,10 @@ public class TestUtils
public static void compareScalars(String expected, String actual) {
assertEquals(expected, actual);
}
+
+ public static void compareScalars(Boolean expected, Boolean actual) {
+ assertEquals(expected, actual);
+ }
public static boolean compareMatrices(HashMap<CellIndex, Double> m1, HashMap<CellIndex, Double> m2,
double tolerance, String name1, String name2)
diff --git a/src/test/java/org/apache/sysds/test/functions/lineage/LineageDebuggerTest.java b/src/test/java/org/apache/sysds/test/functions/lineage/LineageDebuggerTest.java
new file mode 100644
index 0000000..8f0cee5
--- /dev/null
+++ b/src/test/java/org/apache/sysds/test/functions/lineage/LineageDebuggerTest.java
@@ -0,0 +1,236 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysds.test.functions.lineage;
+
+import org.apache.sysds.hops.OptimizerUtils;
+import org.apache.sysds.hops.recompile.Recompiler;
+import org.apache.sysds.runtime.lineage.Lineage;
+import org.apache.sysds.runtime.lineage.LineageDebugger;
+import org.apache.sysds.runtime.lineage.LineageItem;
+import org.apache.sysds.runtime.lineage.LineageParser;
+import org.apache.sysds.test.TestConfiguration;
+import org.apache.sysds.test.TestUtils;
+import org.apache.sysds.utils.Explain;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class LineageDebuggerTest extends LineageBase {
+
+ protected static final String TEST_DIR = "functions/lineage/";
+ protected static final String TEST_NAME1 = "LineageDebugger1";
+ protected String TEST_CLASS_DIR = TEST_DIR + LineageDebuggerTest.class.getSimpleName() + "/";
+
+ protected static final int numRecords = 10;
+ protected static final int numFeatures = 5;
+
+
+ @Override
+ public void setUp() {
+ TestUtils.clearAssertionInformation();
+ addTestConfiguration(TEST_NAME1, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME1));
+ }
+
+ @Test
+ public void testLineageDebuggerNaN() {
+ testLineageDebuggerNaN(TEST_NAME1);
+ }
+
+ @Test
+ public void testLineageDebuggerInf1() {
+ testLineageDebuggerInf(TEST_NAME1, true, false);
+ }
+
+ @Test
+ public void testLineageDebuggerInf2() { testLineageDebuggerInf(TEST_NAME1, false, true); }
+
+ @Test
+ public void testLineageDebuggerInf3() { testLineageDebuggerInf(TEST_NAME1, true, true); }
+
+ @Test
+ public void testLineageDebuggerFirstOccurrence1() { testLineageDebuggerFirstOccurrence(TEST_NAME1); }
+
+ public void testLineageDebuggerNaN(String testname) {
+ boolean old_simplification = OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION;
+ boolean old_sum_product = OptimizerUtils.ALLOW_SUM_PRODUCT_REWRITES;
+
+ try {
+ LOG.debug("------------ BEGIN " + testname + "------------");
+
+ OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION = false;
+ OptimizerUtils.ALLOW_SUM_PRODUCT_REWRITES = false;
+
+ int rows = numRecords;
+ int cols = numFeatures;
+
+ getAndLoadTestConfiguration(testname);
+
+ List<String> proArgs = new ArrayList<>();
+
+ proArgs.add("-stats");
+ proArgs.add("-lineage");
+ proArgs.add("debugger");
+ proArgs.add("-args");
+ proArgs.add(input("X"));
+ proArgs.add(input("Y"));
+ proArgs.add(output("Z"));
+ programArgs = proArgs.toArray(new String[proArgs.size()]);
+
+ fullDMLScriptName = getScript();
+
+ double[][] X = getRandomMatrix(rows, cols, 0, 1, 0.8, -1);
+ writeInputMatrixWithMTD("X", X, true);
+
+ double[][] Y = getRandomMatrix(rows, cols, 0, 1, 0.8, -1);
+ Y[3][3] = Double.NaN;
+ writeInputMatrixWithMTD("Y", Y, true);
+
+ Lineage.resetInternalState();
+ runTest(true, EXCEPTION_NOT_EXPECTED, null, -1);
+
+ String Z_lineage = readDMLLineageFromHDFS("Z");
+ LineageItem Z_li = LineageParser.parseLineageTrace(Z_lineage);
+ TestUtils.compareScalars(Z_lineage, Explain.explain(Z_li));
+
+ TestUtils.compareScalars(true, Z_li.getSpecialValueBit(LineageDebugger.POS_NAN));
+ TestUtils.compareScalars(false, Z_li.getSpecialValueBit(LineageDebugger.POS_NEGATIVE_INFINITY));
+ TestUtils.compareScalars(false, Z_li.getSpecialValueBit(LineageDebugger.POS_POSITIVE_INFINITY));
+ }
+ finally {
+ OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION = old_simplification;
+ OptimizerUtils.ALLOW_SUM_PRODUCT_REWRITES = old_sum_product;
+ Recompiler.reinitRecompiler();
+ }
+ }
+
+ public void testLineageDebuggerInf(String testname, boolean positive, boolean negative) {
+ boolean old_simplification = OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION;
+ boolean old_sum_product = OptimizerUtils.ALLOW_SUM_PRODUCT_REWRITES;
+
+ try {
+ LOG.debug("------------ BEGIN " + testname + "------------");
+
+ OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION = false;
+ OptimizerUtils.ALLOW_SUM_PRODUCT_REWRITES = false;
+
+ int rows = numRecords;
+ int cols = numFeatures;
+
+ getAndLoadTestConfiguration(testname);
+
+ List<String> proArgs = new ArrayList<>();
+
+ proArgs.add("-stats");
+ proArgs.add("-lineage");
+ proArgs.add("debugger");
+ proArgs.add("-args");
+ proArgs.add(input("X"));
+ proArgs.add(input("Y"));
+ proArgs.add(output("Z"));
+ programArgs = proArgs.toArray(new String[proArgs.size()]);
+
+ fullDMLScriptName = getScript();
+
+ double[][] X = getRandomMatrix(rows, cols, 0, 1, 0.8, -1);
+ writeInputMatrixWithMTD("X", X, true);
+
+ double[][] Y = getRandomMatrix(rows, cols, 0, 1, 0.8, -1);
+ if (positive)
+ Y[2][2] = Double.POSITIVE_INFINITY;
+ if (negative)
+ Y[3][3] = Double.NEGATIVE_INFINITY;
+ writeInputMatrixWithMTD("Y", Y, true);
+
+ Lineage.resetInternalState();
+ runTest(true, EXCEPTION_NOT_EXPECTED, null, -1);
+
+ String Z_lineage = readDMLLineageFromHDFS("Z");
+ LineageItem Z_li = LineageParser.parseLineageTrace(Z_lineage);
+ TestUtils.compareScalars(Z_lineage, Explain.explain(Z_li));
+
+ if (positive)
+ TestUtils.compareScalars(true, Z_li.getSpecialValueBit(LineageDebugger.POS_POSITIVE_INFINITY));
+ if (negative)
+ TestUtils.compareScalars(true, Z_li.getSpecialValueBit(LineageDebugger.POS_NEGATIVE_INFINITY));
+ }
+ finally {
+ OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION = old_simplification;
+ OptimizerUtils.ALLOW_SUM_PRODUCT_REWRITES = old_sum_product;
+ Recompiler.reinitRecompiler();
+ }
+ }
+
+ public void testLineageDebuggerFirstOccurrence(String testname) {
+ boolean old_simplification = OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION;
+ boolean old_sum_product = OptimizerUtils.ALLOW_SUM_PRODUCT_REWRITES;
+
+ try {
+ LOG.debug("------------ BEGIN " + testname + "------------");
+
+ OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION = false;
+ OptimizerUtils.ALLOW_SUM_PRODUCT_REWRITES = false;
+
+ int rows = numRecords;
+ int cols = numFeatures;
+
+ getAndLoadTestConfiguration(testname);
+
+ List<String> proArgs = new ArrayList<>();
+
+ proArgs.add("-stats");
+ proArgs.add("-lineage");
+ proArgs.add("debugger");
+ proArgs.add("-args");
+ proArgs.add(input("X"));
+ proArgs.add(input("Y"));
+ proArgs.add(output("Z"));
+ programArgs = proArgs.toArray(new String[proArgs.size()]);
+
+ fullDMLScriptName = getScript();
+
+ double[][] X = getRandomMatrix(rows, cols, 0, 1, 0.8, -1);
+
+ writeInputMatrixWithMTD("X", X, true);
+
+ double[][] Y = getRandomMatrix(rows, cols, 0, 1, 0.8, -1);
+ Y[3][3] = Double.NaN;
+ writeInputMatrixWithMTD("Y", Y, true);
+
+ Lineage.resetInternalState();
+ runTest(true, EXCEPTION_NOT_EXPECTED, null, -1);
+
+ String Z_lineage = readDMLLineageFromHDFS("Z");
+ LineageItem Z_li = LineageParser.parseLineageTrace(Z_lineage);
+ TestUtils.compareScalars(Z_lineage, Explain.explain(Z_li));
+
+ LineageItem firstOccurrence = LineageDebugger.firstOccurrenceOfNR(Z_li, LineageDebugger.POS_NAN);
+ TestUtils.compareScalars(true, firstOccurrence.getSpecialValueBit(LineageDebugger.POS_NAN));
+ for (LineageItem li : firstOccurrence.getInputs())
+ TestUtils.compareScalars(false, li.getSpecialValueBit(LineageDebugger.POS_NAN));
+ }
+ finally {
+ OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION = old_simplification;
+ OptimizerUtils.ALLOW_SUM_PRODUCT_REWRITES = old_sum_product;
+ Recompiler.reinitRecompiler();
+ }
+ }
+
+}
diff --git a/src/test/scripts/functions/lineage/LineageDebugger1.dml b/src/test/scripts/functions/lineage/LineageDebugger1.dml
new file mode 100644
index 0000000..6061b42
--- /dev/null
+++ b/src/test/scripts/functions/lineage/LineageDebugger1.dml
@@ -0,0 +1,39 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+# How to invoke this dml script LineageTrace.dml?
+# Assume LR_HOME is set to the home of the dml script
+# Assume rows = 20 and cols = 20 for X
+# hadoop jar SystemDS.jar -f $LR_HOME/LineageTrace.dml -args "$INPUT_DIR/X" "$OUTPUT_DIR/X" "$OUTPUT_DIR/Y"
+
+X = read($1);
+Y = read($2);
+
+X = X + 3;
+X = X * 5;
+
+X = X * Y;
+
+Z = t(X) %*% X;
+
+# write(X, $3, format="text");
+# write(Y, $4, format="text");
+write(Z, $3, format="text");