You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemds.apache.org by mb...@apache.org on 2024/09/08 14:40:09 UTC
(systemds) branch main updated: [SYSTEMDS-3764] Fix parser-level constant propagation w/ multi-returns
This is an automated email from the ASF dual-hosted git repository.
mboehm7 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/systemds.git
The following commit(s) were added to refs/heads/main by this push:
new b3517cb568 [SYSTEMDS-3764] Fix parser-level constant propagation w/ multi-returns
b3517cb568 is described below
commit b3517cb5689e02a9457aed624d8b1b21fc85d410
Author: Matthias Boehm <mb...@gmail.com>
AuthorDate: Sun Sep 8 16:39:43 2024 +0200
[SYSTEMDS-3764] Fix parser-level constant propagation w/ multi-returns
This patch fixes an old bug of parser-level constant propagation where
updates to scalars through assignments from multi-return function
were not updated the set of constant vars, leading to incorrectly
propagated scalars, constant folding, and utimately wrong results.
This bug was discovered while improving incremental slice line.
---
scripts/builtin/incSliceLine.dml | 3 +-
.../org/apache/sysds/parser/IfStatementBlock.java | 8 +--
.../org/apache/sysds/parser/StatementBlock.java | 4 ++
.../misc/IPAScalarPropagationMulitReturnTest.java | 75 ++++++++++++++++++++++
.../misc/ScalarPropagationMultiReturn.dml | 38 +++++++++++
5 files changed, 122 insertions(+), 6 deletions(-)
diff --git a/scripts/builtin/incSliceLine.dml b/scripts/builtin/incSliceLine.dml
index 72b1390334..19b68ae496 100644
--- a/scripts/builtin/incSliceLine.dml
+++ b/scripts/builtin/incSliceLine.dml
@@ -172,8 +172,7 @@ m_incSliceLine = function(
minsc = -Inf
if( nrow(prevTK) > 0 ) {
prevTK2 = oneHotEncodeUsingOffsets(prevTK, foffb, foffe);
- [minsc2, prevTKC2] = computeLowestPrevTK(prevTK2, X2, totalE, eAvg, alpha)
- minsc = minsc2; #FIXME otherwise -Inf incorrectly propagated
+ [minsc, prevTKC2] = computeLowestPrevTK(prevTK2, X2, totalE, eAvg, alpha)
}
# create and score basic slices (conjunctions of 1 feature)
diff --git a/src/main/java/org/apache/sysds/parser/IfStatementBlock.java b/src/main/java/org/apache/sysds/parser/IfStatementBlock.java
index d34da7557a..77d90bbdab 100644
--- a/src/main/java/org/apache/sysds/parser/IfStatementBlock.java
+++ b/src/main/java/org/apache/sysds/parser/IfStatementBlock.java
@@ -77,12 +77,12 @@ public class IfStatementBlock extends StatementBlock
/////////////////////////////////////////////////////////////////////////////////
// check data type and value type are same for updated variables in both
- // if statement and else statement
+ // if statement and else statement
// (reject conditional data type change)
/////////////////////////////////////////////////////////////////////////////////
for (String updatedVar : this._updated.getVariableNames()){
DataIdentifier origVersion = idsOrigCopy.getVariable(updatedVar);
- DataIdentifier ifVersion = idsIfCopy.getVariable(updatedVar);
+ DataIdentifier ifVersion = idsIfCopy.getVariable(updatedVar);
DataIdentifier elseVersion = idsElseCopy.getVariable(updatedVar);
//data type handling: reject conditional data type change
@@ -90,7 +90,7 @@ public class IfStatementBlock extends StatementBlock
{
if (!ifVersion.getOutput().getDataType().equals(elseVersion.getOutput().getDataType())){
raiseValidateError("IfStatementBlock has unsupported conditional data type change of variable '"+updatedVar+"' in if/else branch.", conditional);
- }
+ }
}
else if( origVersion !=null ) //only if branch exists
{
@@ -99,7 +99,7 @@ public class IfStatementBlock extends StatementBlock
}
}
- //value type handling
+ //value type handling
if (ifVersion != null && elseVersion != null && !ifVersion.getOutput().getValueType().equals(elseVersion.getOutput().getValueType())){
LOG.warn(elseVersion.printWarningLocation() + "Variable " + elseVersion.getName() + " defined with different value type in if and else clause.");
}
diff --git a/src/main/java/org/apache/sysds/parser/StatementBlock.java b/src/main/java/org/apache/sysds/parser/StatementBlock.java
index 11b5a52648..cc37b252c6 100644
--- a/src/main/java/org/apache/sysds/parser/StatementBlock.java
+++ b/src/main/java/org/apache/sysds/parser/StatementBlock.java
@@ -1099,6 +1099,10 @@ public class StatementBlock extends LiveVariableAnalysis implements ParseInfo
ids.addVariable(targetList.get(j).getName(), (DataIdentifier)outputs[j]);
}
}
+
+ // remove updated constant vars (for correctness)
+ for(DataIdentifier target : targetList)
+ currConstVars.remove(target.getName());
}
public void setStatementFormatType(OutputStatement s, boolean conditionalValidate)
diff --git a/src/test/java/org/apache/sysds/test/functions/misc/IPAScalarPropagationMulitReturnTest.java b/src/test/java/org/apache/sysds/test/functions/misc/IPAScalarPropagationMulitReturnTest.java
new file mode 100644
index 0000000000..af20e12408
--- /dev/null
+++ b/src/test/java/org/apache/sysds/test/functions/misc/IPAScalarPropagationMulitReturnTest.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysds.test.functions.misc;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.apache.sysds.hops.OptimizerUtils;
+import org.apache.sysds.test.AutomatedTestBase;
+import org.apache.sysds.test.TestConfiguration;
+import org.apache.sysds.test.TestUtils;
+
+public class IPAScalarPropagationMulitReturnTest extends AutomatedTestBase
+{
+ private final static String TEST_NAME1 = "ScalarPropagationMultiReturn";
+ private final static String TEST_DIR = "functions/misc/";
+ private final static String TEST_CLASS_DIR = TEST_DIR + IPAScalarPropagationMulitReturnTest.class.getSimpleName() + "/";
+
+ @Override
+ public void setUp() {
+ TestUtils.clearAssertionInformation();
+ addTestConfiguration( TEST_NAME1, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME1, new String[]{"R"}) );
+ }
+
+ @Test
+ public void testScalarPropagationNoIPA() {
+ runIPAScalarVariablePropagationTest( TEST_NAME1, false );
+ }
+
+ @Test
+ public void testScalarPropagationIPA() {
+ runIPAScalarVariablePropagationTest( TEST_NAME1, true );
+ }
+
+ private void runIPAScalarVariablePropagationTest( String testname, boolean IPA )
+ {
+ boolean oldFlagIPA = OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS;
+
+ try {
+ TestConfiguration config = getTestConfiguration(testname);
+ loadTestConfiguration(config);
+
+ String HOME = SCRIPT_DIR + TEST_DIR;
+ fullDMLScriptName = HOME + testname + ".dml";
+ programArgs = new String[]{"-explain","-args", output("R") };
+ OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS = IPA;
+
+ //run test, incl expected MR jobs (in case if IPA 0 due to scalar propagation)
+ runTest(true, false, null, IPA ? 0 : 35);
+
+ double[][] ret = TestUtils.convertHashMapToDoubleArray(
+ readDMLMatrixFromOutputDir("R"), 1, 1);
+ Assert.assertEquals(Double.valueOf(2), Double.valueOf(ret[0][0]));
+ }
+ finally {
+ OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS = oldFlagIPA;
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/test/scripts/functions/misc/ScalarPropagationMultiReturn.dml b/src/test/scripts/functions/misc/ScalarPropagationMultiReturn.dml
new file mode 100644
index 0000000000..023cbcfbac
--- /dev/null
+++ b/src/test/scripts/functions/misc/ScalarPropagationMultiReturn.dml
@@ -0,0 +1,38 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+
+foo = function(matrix[double] A) return (Double minsc, matrix[double] B) {
+ while(FALSE) {} # prevent inlining
+ minsc = min(A);
+ B = A + 7;
+}
+
+X = seq(1,7);
+minsc = -Inf;
+if( avg(X) <= 5 ) {
+ [minsc, X] = foo(X)
+}
+
+# bug: minsc was incorrectly propagated as -Inf, leading to -Inf instead of 2
+R = as.matrix(minsc + 1);
+write(R, $1);
+