You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemds.apache.org by mb...@apache.org on 2022/05/07 22:35:14 UTC
[systemds] branch main updated: [SYSTEMDS-3370] Fix size propagation of list operations
This is an automated email from the ASF dual-hosted git repository.
mboehm7 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/systemds.git
The following commit(s) were added to refs/heads/main by this push:
new 5f98eb9602 [SYSTEMDS-3370] Fix size propagation of list operations
5f98eb9602 is described below
commit 5f98eb9602371ce45ad19889f8d4c28ece32a7ae
Author: Matthias Boehm <mb...@gmail.com>
AuthorDate: Sun May 8 00:34:54 2022 +0200
[SYSTEMDS-3370] Fix size propagation of list operations
This patch adds dedicated tests for checking the size propagation of
list operations, and fixes related issues in the parser and hops.
The ignored test requires a parser/runtime extension for list flattening
e.g., via as.list() similar to as.matrix()/as.scalar().
---
src/main/java/org/apache/sysds/hops/BinaryOp.java | 9 ++-
.../sysds/parser/BuiltinFunctionExpression.java | 8 +-
.../functions/misc/AsBooleanVsAsLogicalTest.java | 3 +-
.../test/functions/misc/ListAppendSizeTest.java | 91 ++++++++++++++++++++++
src/test/scripts/functions/lineage/MiscProbe1.dml | 22 +++++-
src/test/scripts/functions/lineage/MiscProbe2.dml | 22 +++++-
.../scripts/functions/misc/ListAppendSize1.dml | 26 +++++++
.../scripts/functions/misc/ListAppendSize2.dml | 27 +++++++
.../scripts/functions/misc/ListAppendSize3.dml | 29 +++++++
.../scripts/functions/misc/ListAppendSize4.dml | 32 ++++++++
10 files changed, 261 insertions(+), 8 deletions(-)
diff --git a/src/main/java/org/apache/sysds/hops/BinaryOp.java b/src/main/java/org/apache/sysds/hops/BinaryOp.java
index 1151135ae0..20178afc73 100644
--- a/src/main/java/org/apache/sysds/hops/BinaryOp.java
+++ b/src/main/java/org/apache/sysds/hops/BinaryOp.java
@@ -951,12 +951,17 @@ public class BinaryOp extends MultiThreadedHop {
DataType dt1 = input1.getDataType();
DataType dt2 = input2.getDataType();
- if ( getDataType() == DataType.SCALAR )
- {
+ if ( getDataType() == DataType.SCALAR ) {
//do nothing always known
setDim1(0);
setDim2(0);
}
+ else if ( getDataType() == DataType.LIST ) {
+ if( input1.getDataType().isList() && input1.rowsKnown() ) {
+ setDim1(input1.getDim1() + 1);
+ setDim2(1); //always col-vector
+ }
+ }
else //MATRIX OUTPUT
{
//TODO quantile
diff --git a/src/main/java/org/apache/sysds/parser/BuiltinFunctionExpression.java b/src/main/java/org/apache/sysds/parser/BuiltinFunctionExpression.java
index ec5b80b48f..8bbc5be88d 100644
--- a/src/main/java/org/apache/sysds/parser/BuiltinFunctionExpression.java
+++ b/src/main/java/org/apache/sysds/parser/BuiltinFunctionExpression.java
@@ -361,13 +361,15 @@ public class BuiltinFunctionExpression extends DataIdentifier
DataIdentifier out1 = (DataIdentifier) getOutputs()[0];
DataIdentifier out2 = (DataIdentifier) getOutputs()[1];
- // Output1 - Eigen Values
+ // Output1 - list after removal
+ long nrow = getFirstExpr().getOutput().getDim1() > 0 ?
+ getFirstExpr().getOutput().getDim1() + 1 : -1;
out1.setDataType(DataType.LIST);
out1.setValueType(getFirstExpr().getOutput().getValueType());
- out1.setDimensions(getFirstExpr().getOutput().getDim1()-1, 1);
+ out1.setDimensions(nrow, 1);
out1.setBlocksize(getFirstExpr().getOutput().getBlocksize());
- // Output2 - Eigen Vectors
+ // Output2 - list of removed element
out2.setDataType(DataType.LIST);
out2.setValueType(getFirstExpr().getOutput().getValueType());
out2.setDimensions(1, 1);
diff --git a/src/test/java/org/apache/sysds/test/functions/misc/AsBooleanVsAsLogicalTest.java b/src/test/java/org/apache/sysds/test/functions/misc/AsBooleanVsAsLogicalTest.java
index 0b1eab8330..858f5f1a0e 100644
--- a/src/test/java/org/apache/sysds/test/functions/misc/AsBooleanVsAsLogicalTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/misc/AsBooleanVsAsLogicalTest.java
@@ -46,10 +46,11 @@ public class AsBooleanVsAsLogicalTest extends AutomatedTestBase {
public void testPrintNotExpressionTest() {
TestConfiguration config = getTestConfiguration(TEST_NAME1);
loadTestConfiguration(config);
- String HOME = SCRIPT_DIR + TEST_DIR;
+ String HOME = SCRIPT_DIR + TEST_DIR;
fullDMLScriptName = HOME + TEST_NAME1 + ".dml";
try{
programArgs = new String[]{};
+ setOutputBuffering(true);
String out = runTest(null).toString();
LOG.debug(out);
assertTrue(out.contains("TRUE\nFALSE\nFALSE"));
diff --git a/src/test/java/org/apache/sysds/test/functions/misc/ListAppendSizeTest.java b/src/test/java/org/apache/sysds/test/functions/misc/ListAppendSizeTest.java
new file mode 100644
index 0000000000..c79753ab5a
--- /dev/null
+++ b/src/test/java/org/apache/sysds/test/functions/misc/ListAppendSizeTest.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysds.test.functions.misc;
+
+import org.junit.Assert;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.apache.sysds.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysds.common.Types.ExecMode;
+import org.apache.sysds.common.Types.ExecType;
+import org.apache.sysds.test.AutomatedTestBase;
+import org.apache.sysds.test.TestConfiguration;
+import org.apache.sysds.test.TestUtils;
+
+public class ListAppendSizeTest extends AutomatedTestBase
+{
+ private static final String TEST_NAME1 = "ListAppendSize1";
+ private static final String TEST_NAME2 = "ListAppendSize2";
+ private static final String TEST_NAME3 = "ListAppendSize3";
+ private static final String TEST_NAME4 = "ListAppendSize4";
+
+ private static final String TEST_DIR = "functions/misc/";
+ private static final String TEST_CLASS_DIR = TEST_DIR + ListAppendSizeTest.class.getSimpleName() + "/";
+
+ @Override
+ public void setUp() {
+ TestUtils.clearAssertionInformation();
+ addTestConfiguration( TEST_NAME1, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME1, new String[] { "R" }) );
+ addTestConfiguration( TEST_NAME2, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME2, new String[] { "R" }) );
+ addTestConfiguration( TEST_NAME3, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME3, new String[] { "R" }) );
+ addTestConfiguration( TEST_NAME4, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME4, new String[] { "R" }) );
+ }
+
+ @Test
+ public void testListAppendSize1CP() {
+ runListAppendSize(TEST_NAME1, ExecType.CP, 4);
+ }
+
+ @Test
+ public void testListAppendSize2CP() {
+ runListAppendSize(TEST_NAME2, ExecType.CP, 3);
+ }
+
+ @Test
+ public void testListAppendSize3CP() {
+ runListAppendSize(TEST_NAME3, ExecType.CP, 2);
+ }
+
+ @Test
+ @Ignore //TODO support for as.list unnesting
+ public void testListAppendSize4CP() {
+ runListAppendSize(TEST_NAME4, ExecType.CP, 4);
+ }
+
+ private void runListAppendSize(String testname, ExecType type, int expected) {
+ ExecMode platformOld = setExecMode(type);
+
+ try {
+ getAndLoadTestConfiguration(testname);
+
+ String HOME = SCRIPT_DIR + TEST_DIR;
+ fullDMLScriptName = HOME + testname + ".dml";
+ programArgs = new String[]{ "-stats","-explain","-args", output("R") };
+
+ //run test
+ runTest(true, false, null, -1);
+ double ret = readDMLMatrixFromOutputDir("R").get(new CellIndex(1,1));
+ Assert.assertEquals(Integer.valueOf(expected), Integer.valueOf((int)ret));
+ }
+ finally {
+ resetExecMode(platformOld);
+ }
+ }
+}
diff --git a/src/test/scripts/functions/lineage/MiscProbe1.dml b/src/test/scripts/functions/lineage/MiscProbe1.dml
index 5f19f14fdd..ec189f0735 100644
--- a/src/test/scripts/functions/lineage/MiscProbe1.dml
+++ b/src/test/scripts/functions/lineage/MiscProbe1.dml
@@ -1,6 +1,26 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
X = rand(rows=10, cols=8, seed=1234);
-n = 200000;
+n = 70000;
for(counter in 1:n) { # create lineage trace
X = X + 0.1;
}
diff --git a/src/test/scripts/functions/lineage/MiscProbe2.dml b/src/test/scripts/functions/lineage/MiscProbe2.dml
index 4204c59469..b7d325ae48 100644
--- a/src/test/scripts/functions/lineage/MiscProbe2.dml
+++ b/src/test/scripts/functions/lineage/MiscProbe2.dml
@@ -1,6 +1,26 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
X = rand(rows=10, cols=8, seed=1234);
-n = 200000;
+n = 70000;
for(counter in 1:n) { # create lineage trace
X = 0.1 + X;
}
diff --git a/src/test/scripts/functions/misc/ListAppendSize1.dml b/src/test/scripts/functions/misc/ListAppendSize1.dml
new file mode 100644
index 0000000000..21e9289a6f
--- /dev/null
+++ b/src/test/scripts/functions/misc/ListAppendSize1.dml
@@ -0,0 +1,26 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+l1 = list(1, 2, 3)
+l1 = append(l1, 4)
+S = as.matrix(length(l1))
+
+write(S, $1)
\ No newline at end of file
diff --git a/src/test/scripts/functions/misc/ListAppendSize2.dml b/src/test/scripts/functions/misc/ListAppendSize2.dml
new file mode 100644
index 0000000000..7d9b0c33b6
--- /dev/null
+++ b/src/test/scripts/functions/misc/ListAppendSize2.dml
@@ -0,0 +1,27 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+l1 = list(1, 2, 3)
+l1 = append(l1, 4)
+[l2,i] = remove(l1, 1)
+S = as.matrix(length(l2))
+
+write(S, $1)
\ No newline at end of file
diff --git a/src/test/scripts/functions/misc/ListAppendSize3.dml b/src/test/scripts/functions/misc/ListAppendSize3.dml
new file mode 100644
index 0000000000..a74c7bc17b
--- /dev/null
+++ b/src/test/scripts/functions/misc/ListAppendSize3.dml
@@ -0,0 +1,29 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+l1 = list(1, 2, 3)
+l2 = list()
+l2 = append(l2, l1)
+l3 = l2[1]
+l4 = append(l3, 12)
+S = as.matrix(length(l4))
+
+write(S, $1)
diff --git a/src/test/scripts/functions/misc/ListAppendSize4.dml b/src/test/scripts/functions/misc/ListAppendSize4.dml
new file mode 100644
index 0000000000..0720051f77
--- /dev/null
+++ b/src/test/scripts/functions/misc/ListAppendSize4.dml
@@ -0,0 +1,32 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+l1 = list(1, 2, 3)
+l2 = list()
+l2 = append(l2, l1)
+l3 = as.list(l2[1]) # TODO
+l4 = append(l3, 12)
+
+print(toString(l4))
+print(length(l4))
+
+S = as.matrix(length(l4))
+write(S, $1)