You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2018/05/18 01:12:13 UTC
systemml git commit: [SYSTEMML-2327] Fix missing list support in
parfor, incl tests
Repository: systemml
Updated Branches:
refs/heads/master a5a4d4d33 -> 79a5e80f6
[SYSTEMML-2327] Fix missing list support in parfor, incl tests
This patch adds tests with lists and named lists for parfor loops and
dependency analysis and fixes a minor issue with unbounded scoping of
temporary list intermediates.
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/79a5e80f
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/79a5e80f
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/79a5e80f
Branch: refs/heads/master
Commit: 79a5e80f62d387af8ad3a3cf1967ca8adc9882b9
Parents: a5a4d4d
Author: Matthias Boehm <mb...@gmail.com>
Authored: Thu May 17 18:13:43 2018 -0700
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Thu May 17 18:13:43 2018 -0700
----------------------------------------------------------------------
.../controlprogram/ParForProgramBlock.java | 5 +++
.../functions/misc/ListAndStructTest.java | 25 +++++++++++-
.../parfor/ParForDependencyAnalysisTest.java | 11 +++++
.../scripts/functions/misc/ListNamedParfor.R | 42 ++++++++++++++++++++
.../scripts/functions/misc/ListNamedParfor.dml | 36 +++++++++++++++++
.../scripts/functions/misc/ListUnnamedParfor.R | 42 ++++++++++++++++++++
.../functions/misc/ListUnnamedParfor.dml | 37 +++++++++++++++++
src/test/scripts/functions/parfor/parfor54a.dml | 29 ++++++++++++++
src/test/scripts/functions/parfor/parfor54b.dml | 29 ++++++++++++++
src/test/scripts/functions/parfor/parfor54c.dml | 29 ++++++++++++++
10 files changed, 284 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/79a5e80f/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java b/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java
index 732e59d..368737b 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java
@@ -26,6 +26,7 @@ import java.io.Serializable;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
+import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -103,6 +104,7 @@ import org.apache.sysml.runtime.instructions.cp.BooleanObject;
import org.apache.sysml.runtime.instructions.cp.Data;
import org.apache.sysml.runtime.instructions.cp.DoubleObject;
import org.apache.sysml.runtime.instructions.cp.IntObject;
+import org.apache.sysml.runtime.instructions.cp.ListObject;
import org.apache.sysml.runtime.instructions.cp.StringObject;
import org.apache.sysml.runtime.instructions.cp.VariableCPInstruction;
import org.apache.sysml.runtime.io.IOUtilFunctions;
@@ -1274,6 +1276,9 @@ public class ParForProgramBlock extends ForProgramBlock
//currently we do not create any unscoped matrix or frame outputs
//because metadata (e.g., outputinfo) not known at this place.
break;
+ case LIST:
+ dataObj = new ListObject(Collections.emptyList());
+ break;
case UNKNOWN:
break;
default:
http://git-wip-us.apache.org/repos/asf/systemml/blob/79a5e80f/src/test/java/org/apache/sysml/test/integration/functions/misc/ListAndStructTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/misc/ListAndStructTest.java b/src/test/java/org/apache/sysml/test/integration/functions/misc/ListAndStructTest.java
index 4c33847..b49f84c 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/misc/ListAndStructTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/misc/ListAndStructTest.java
@@ -37,7 +37,8 @@ public class ListAndStructTest extends AutomatedTestBase
private static final String TEST_NAME2 = "ListNamed";
private static final String TEST_NAME3 = "ListUnnamedFun";
private static final String TEST_NAME4 = "ListNamedFun";
-
+ private static final String TEST_NAME5 = "ListUnnamedParfor";
+ private static final String TEST_NAME6 = "ListNamedParfor";
private static final String TEST_DIR = "functions/misc/";
private static final String TEST_CLASS_DIR = TEST_DIR + ListAndStructTest.class.getSimpleName() + "/";
@@ -49,6 +50,8 @@ public class ListAndStructTest extends AutomatedTestBase
addTestConfiguration( TEST_NAME2, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME2, new String[] { "R" }) );
addTestConfiguration( TEST_NAME3, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME3, new String[] { "R" }) );
addTestConfiguration( TEST_NAME4, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME4, new String[] { "R" }) );
+ addTestConfiguration( TEST_NAME5, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME5, new String[] { "R" }) );
+ addTestConfiguration( TEST_NAME6, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME6, new String[] { "R" }) );
}
@Test
@@ -91,6 +94,26 @@ public class ListAndStructTest extends AutomatedTestBase
runListStructTest(TEST_NAME4, true);
}
+ @Test
+ public void testListUnnamedParFor() {
+ runListStructTest(TEST_NAME5, false);
+ }
+
+ @Test
+ public void testListUnnamedParForRewrites() {
+ runListStructTest(TEST_NAME5, true);
+ }
+
+ @Test
+ public void testListNamedParFor() {
+ runListStructTest(TEST_NAME6, false);
+ }
+
+ @Test
+ public void testListNamedParForRewrites() {
+ runListStructTest(TEST_NAME6, true);
+ }
+
private void runListStructTest(String testname, boolean rewrites)
{
boolean oldFlag = OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION;
http://git-wip-us.apache.org/repos/asf/systemml/blob/79a5e80f/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForDependencyAnalysisTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForDependencyAnalysisTest.java b/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForDependencyAnalysisTest.java
index 26b3fc6..97f253b 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForDependencyAnalysisTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForDependencyAnalysisTest.java
@@ -66,6 +66,8 @@ import org.junit.Test;
* 49a: dep, 49b: dep
* * accumulators
* 53a: no, 53b dep, 53c dep, 53d dep, 53e dep
+ * * lists
+ * 54a: no, 54b: dep, 54c: dep
*/
public class ParForDependencyAnalysisTest extends AutomatedTestBase
{
@@ -316,6 +318,15 @@ public class ParForDependencyAnalysisTest extends AutomatedTestBase
@Test
public void testDependencyAnalysis53e() { runTest("parfor53e.dml", true); }
+ @Test
+ public void testDependencyAnalysis54a() { runTest("parfor54a.dml", false); }
+
+ @Test
+ public void testDependencyAnalysis54b() { runTest("parfor54b.dml", true); }
+
+ @Test
+ public void testDependencyAnalysis54c() { runTest("parfor54c.dml", true); }
+
private void runTest( String scriptFilename, boolean expectedException ) {
boolean raisedException = false;
http://git-wip-us.apache.org/repos/asf/systemml/blob/79a5e80f/src/test/scripts/functions/misc/ListNamedParfor.R
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/misc/ListNamedParfor.R b/src/test/scripts/functions/misc/ListNamedParfor.R
new file mode 100644
index 0000000..abf01be
--- /dev/null
+++ b/src/test/scripts/functions/misc/ListNamedParfor.R
@@ -0,0 +1,42 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+
+args <- commandArgs(TRUE)
+options(digits=22)
+library("Matrix")
+
+A = matrix(1, 10, 10);
+B = matrix(2, 10, 10);
+c = 3;
+D = matrix(4, 10, 10);
+
+X = list(a=A, b=B, c=c, d=D, e=matrix(5, 3, 3), f=6);
+
+for( i in 1:length(X) ) {
+ tmp = X[i]
+ if( !exists("tmp") )
+ print("ERROR: non-existing entry "+i );
+}
+
+R = as.matrix(sum(as.matrix(X[['e']])));
+
+writeMM(as(R, "CsparseMatrix"), paste(args[1], "R", sep=""));
http://git-wip-us.apache.org/repos/asf/systemml/blob/79a5e80f/src/test/scripts/functions/misc/ListNamedParfor.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/misc/ListNamedParfor.dml b/src/test/scripts/functions/misc/ListNamedParfor.dml
new file mode 100644
index 0000000..a00df0d
--- /dev/null
+++ b/src/test/scripts/functions/misc/ListNamedParfor.dml
@@ -0,0 +1,36 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+A = matrix(1, 10, 10);
+B = matrix(2, 10, 10);
+c = 3;
+D = matrix(4, 10, 10);
+
+X = list(a=A, b=B, c=c, d=D, e=matrix(5, 3, 3), f=6);
+for( i in 1:length(X) ) {
+ tmp = X[i];
+ if( !exists(tmp) )
+ print("ERROR: non-existing entry "+i );
+}
+
+R = as.matrix(sum(as.matrix(X['e'])));
+
+write(R, $1);
http://git-wip-us.apache.org/repos/asf/systemml/blob/79a5e80f/src/test/scripts/functions/misc/ListUnnamedParfor.R
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/misc/ListUnnamedParfor.R b/src/test/scripts/functions/misc/ListUnnamedParfor.R
new file mode 100644
index 0000000..ede3c2b
--- /dev/null
+++ b/src/test/scripts/functions/misc/ListUnnamedParfor.R
@@ -0,0 +1,42 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+
+args <- commandArgs(TRUE)
+options(digits=22)
+library("Matrix")
+
+A = matrix(1, 10, 10);
+B = matrix(2, 10, 10);
+c = 3;
+D = matrix(4, 10, 10);
+
+X = list(A, B, c, D, matrix(5, 3, 3), 6);
+
+for( i in 1:length(X) ) {
+ tmp = X[i]
+ if( !exists("tmp") )
+ print("ERROR: non-existing entry "+i );
+}
+
+R = as.matrix(sum(as.matrix(X[[5]])));
+
+writeMM(as(R, "CsparseMatrix"), paste(args[1], "R", sep=""));
http://git-wip-us.apache.org/repos/asf/systemml/blob/79a5e80f/src/test/scripts/functions/misc/ListUnnamedParfor.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/misc/ListUnnamedParfor.dml b/src/test/scripts/functions/misc/ListUnnamedParfor.dml
new file mode 100644
index 0000000..9d38517
--- /dev/null
+++ b/src/test/scripts/functions/misc/ListUnnamedParfor.dml
@@ -0,0 +1,37 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+A = matrix(1, 10, 10);
+B = matrix(2, 10, 10);
+c = 3;
+D = matrix(4, 10, 10);
+
+X = list(A, B, c, D, matrix(5, 3, 3), 6);
+
+parfor( i in 1:length(X) ) {
+ tmp = X[i];
+ if( !exists(tmp) )
+ print("ERROR: non-existing entry "+i );
+}
+
+R = as.matrix(sum(as.matrix(X[5])));
+
+write(R, $1);
http://git-wip-us.apache.org/repos/asf/systemml/blob/79a5e80f/src/test/scripts/functions/parfor/parfor54a.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/parfor/parfor54a.dml b/src/test/scripts/functions/parfor/parfor54a.dml
new file mode 100644
index 0000000..1d3a75e
--- /dev/null
+++ b/src/test/scripts/functions/parfor/parfor54a.dml
@@ -0,0 +1,29 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+
+A = matrix(7, rows=2, cols=2);
+B = matrix(3, rows=2, cols=2);
+C = list(A, B);
+parfor( i in 1:2 ) {
+ print(sum(as.matrix(C[i])));
+}
+print(sum(as.matrix(C[1])));
http://git-wip-us.apache.org/repos/asf/systemml/blob/79a5e80f/src/test/scripts/functions/parfor/parfor54b.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/parfor/parfor54b.dml b/src/test/scripts/functions/parfor/parfor54b.dml
new file mode 100644
index 0000000..3e2c6e8
--- /dev/null
+++ b/src/test/scripts/functions/parfor/parfor54b.dml
@@ -0,0 +1,29 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+
+A = matrix(7, rows=2, cols=2);
+B = matrix(3, rows=2, cols=2);
+C = list(A, B);
+parfor( i in 1:2 ) {
+ C[i] = as.matrix(C[i])+7;
+}
+print(sum(as.matrix(C[1])));
http://git-wip-us.apache.org/repos/asf/systemml/blob/79a5e80f/src/test/scripts/functions/parfor/parfor54c.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/parfor/parfor54c.dml b/src/test/scripts/functions/parfor/parfor54c.dml
new file mode 100644
index 0000000..b4f96d8
--- /dev/null
+++ b/src/test/scripts/functions/parfor/parfor54c.dml
@@ -0,0 +1,29 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+
+A = matrix(7, rows=2, cols=2);
+B = matrix(3, rows=2, cols=2);
+C = list(A, B);
+parfor( i in 1:2 ) {
+ C = list(as.matrix(C[i])+7);
+}
+print(sum(as.matrix(C[1])));