You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemds.apache.org by mb...@apache.org on 2024/09/08 14:40:09 UTC

(systemds) branch main updated: [SYSTEMDS-3764] Fix parser-level constant propagation w/ multi-returns

This is an automated email from the ASF dual-hosted git repository.

mboehm7 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/systemds.git


The following commit(s) were added to refs/heads/main by this push:
     new b3517cb568 [SYSTEMDS-3764] Fix parser-level constant propagation w/ multi-returns
b3517cb568 is described below

commit b3517cb5689e02a9457aed624d8b1b21fc85d410
Author: Matthias Boehm <mb...@gmail.com>
AuthorDate: Sun Sep 8 16:39:43 2024 +0200

    [SYSTEMDS-3764] Fix parser-level constant propagation w/ multi-returns
    
    This patch fixes an old bug of parser-level constant propagation where
    updates to scalars through assignments from multi-return function
    were not updated the set of constant vars, leading to incorrectly
    propagated scalars, constant folding, and utimately wrong results.
    
    This bug was discovered while improving incremental slice line.
---
 scripts/builtin/incSliceLine.dml                   |  3 +-
 .../org/apache/sysds/parser/IfStatementBlock.java  |  8 +--
 .../org/apache/sysds/parser/StatementBlock.java    |  4 ++
 .../misc/IPAScalarPropagationMulitReturnTest.java  | 75 ++++++++++++++++++++++
 .../misc/ScalarPropagationMultiReturn.dml          | 38 +++++++++++
 5 files changed, 122 insertions(+), 6 deletions(-)

diff --git a/scripts/builtin/incSliceLine.dml b/scripts/builtin/incSliceLine.dml
index 72b1390334..19b68ae496 100644
--- a/scripts/builtin/incSliceLine.dml
+++ b/scripts/builtin/incSliceLine.dml
@@ -172,8 +172,7 @@ m_incSliceLine = function(
   minsc = -Inf
   if( nrow(prevTK) > 0 ) {
     prevTK2 = oneHotEncodeUsingOffsets(prevTK, foffb, foffe);
-    [minsc2, prevTKC2] = computeLowestPrevTK(prevTK2, X2, totalE, eAvg, alpha)
-    minsc = minsc2; #FIXME otherwise -Inf incorrectly propagated
+    [minsc, prevTKC2] = computeLowestPrevTK(prevTK2, X2, totalE, eAvg, alpha)
   }
 
   # create and score basic slices (conjunctions of 1 feature)
diff --git a/src/main/java/org/apache/sysds/parser/IfStatementBlock.java b/src/main/java/org/apache/sysds/parser/IfStatementBlock.java
index d34da7557a..77d90bbdab 100644
--- a/src/main/java/org/apache/sysds/parser/IfStatementBlock.java
+++ b/src/main/java/org/apache/sysds/parser/IfStatementBlock.java
@@ -77,12 +77,12 @@ public class IfStatementBlock extends StatementBlock
 		
 		/////////////////////////////////////////////////////////////////////////////////
 		//  check data type and value type are same for updated variables in both 
-		//	if statement and else statement
+		//  if statement and else statement
 		//  (reject conditional data type change)
 		/////////////////////////////////////////////////////////////////////////////////
 		for (String updatedVar : this._updated.getVariableNames()){
 			DataIdentifier origVersion  = idsOrigCopy.getVariable(updatedVar);
-			DataIdentifier ifVersion 	= idsIfCopy.getVariable(updatedVar);
+			DataIdentifier ifVersion    = idsIfCopy.getVariable(updatedVar);
 			DataIdentifier elseVersion  = idsElseCopy.getVariable(updatedVar);
 			
 			//data type handling: reject conditional data type change
@@ -90,7 +90,7 @@ public class IfStatementBlock extends StatementBlock
 			{
 				if (!ifVersion.getOutput().getDataType().equals(elseVersion.getOutput().getDataType())){
 					raiseValidateError("IfStatementBlock has unsupported conditional data type change of variable '"+updatedVar+"' in if/else branch.", conditional);
-				}	
+				}
 			}
 			else if( origVersion !=null ) //only if branch exists
 			{
@@ -99,7 +99,7 @@ public class IfStatementBlock extends StatementBlock
 				}
 			}
 			
-			//value type handling		
+			//value type handling
 			if (ifVersion != null && elseVersion != null && !ifVersion.getOutput().getValueType().equals(elseVersion.getOutput().getValueType())){
 				LOG.warn(elseVersion.printWarningLocation() + "Variable " + elseVersion.getName() + " defined with different value type in if and else clause.");
 			}
diff --git a/src/main/java/org/apache/sysds/parser/StatementBlock.java b/src/main/java/org/apache/sysds/parser/StatementBlock.java
index 11b5a52648..cc37b252c6 100644
--- a/src/main/java/org/apache/sysds/parser/StatementBlock.java
+++ b/src/main/java/org/apache/sysds/parser/StatementBlock.java
@@ -1099,6 +1099,10 @@ public class StatementBlock extends LiveVariableAnalysis implements ParseInfo
 				ids.addVariable(targetList.get(j).getName(), (DataIdentifier)outputs[j]);
 			}
 		}
+		
+		// remove updated constant vars (for correctness)
+		for(DataIdentifier target : targetList)
+			currConstVars.remove(target.getName());
 	}
 	
 	public void setStatementFormatType(OutputStatement s, boolean conditionalValidate)
diff --git a/src/test/java/org/apache/sysds/test/functions/misc/IPAScalarPropagationMulitReturnTest.java b/src/test/java/org/apache/sysds/test/functions/misc/IPAScalarPropagationMulitReturnTest.java
new file mode 100644
index 0000000000..af20e12408
--- /dev/null
+++ b/src/test/java/org/apache/sysds/test/functions/misc/IPAScalarPropagationMulitReturnTest.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysds.test.functions.misc;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.apache.sysds.hops.OptimizerUtils;
+import org.apache.sysds.test.AutomatedTestBase;
+import org.apache.sysds.test.TestConfiguration;
+import org.apache.sysds.test.TestUtils;
+
+public class IPAScalarPropagationMulitReturnTest extends AutomatedTestBase 
+{
+	private final static String TEST_NAME1 = "ScalarPropagationMultiReturn";
+	private final static String TEST_DIR = "functions/misc/";
+	private final static String TEST_CLASS_DIR = TEST_DIR + IPAScalarPropagationMulitReturnTest.class.getSimpleName() + "/";
+	
+	@Override
+	public void setUp() {
+		TestUtils.clearAssertionInformation();
+		addTestConfiguration( TEST_NAME1, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME1, new String[]{"R"})   );
+	}
+
+	@Test
+	public void testScalarPropagationNoIPA() {
+		runIPAScalarVariablePropagationTest( TEST_NAME1, false );
+	}
+	
+	@Test
+	public void testScalarPropagationIPA() {
+		runIPAScalarVariablePropagationTest( TEST_NAME1, true );
+	}
+	
+	private void runIPAScalarVariablePropagationTest( String testname, boolean IPA )
+	{
+		boolean oldFlagIPA = OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS;
+		
+		try {
+			TestConfiguration config = getTestConfiguration(testname);
+			loadTestConfiguration(config);
+			
+			String HOME = SCRIPT_DIR + TEST_DIR;
+			fullDMLScriptName = HOME + testname + ".dml";
+			programArgs = new String[]{"-explain","-args", output("R") };
+			OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS = IPA;
+
+			//run test, incl expected MR jobs (in case if IPA 0 due to scalar propagation)
+			runTest(true, false, null, IPA ? 0 : 35);
+			
+			double[][] ret = TestUtils.convertHashMapToDoubleArray(
+				readDMLMatrixFromOutputDir("R"), 1, 1);
+			Assert.assertEquals(Double.valueOf(2), Double.valueOf(ret[0][0]));
+		}
+		finally {
+			OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS = oldFlagIPA;
+		}
+	}
+}
\ No newline at end of file
diff --git a/src/test/scripts/functions/misc/ScalarPropagationMultiReturn.dml b/src/test/scripts/functions/misc/ScalarPropagationMultiReturn.dml
new file mode 100644
index 0000000000..023cbcfbac
--- /dev/null
+++ b/src/test/scripts/functions/misc/ScalarPropagationMultiReturn.dml
@@ -0,0 +1,38 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+
+foo = function(matrix[double] A) return (Double minsc, matrix[double] B) {
+  while(FALSE) {} # prevent inlining
+  minsc = min(A);
+  B = A + 7;
+}
+
+X = seq(1,7);
+minsc = -Inf;
+if( avg(X) <= 5 ) {
+  [minsc, X] = foo(X)
+}
+
+# bug: minsc was incorrectly propagated as -Inf, leading to -Inf instead of 2
+R = as.matrix(minsc + 1);
+write(R, $1);
+