You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2017/11/05 01:56:28 UTC
[1/3] systemml git commit: [HOTFIX] Fix JMLC tests (write/read order,
tmp dir, fixed seeds)
Repository: systemml
Updated Branches:
refs/heads/master ae819a253 -> c0b6ef5ca
[HOTFIX] Fix JMLC tests (write/read order, tmp dir, fixed seeds)
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/aaa94818
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/aaa94818
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/aaa94818
Branch: refs/heads/master
Commit: aaa94818e4104c5b711507b759b372426ca1198c
Parents: ae819a2
Author: Matthias Boehm <mb...@gmail.com>
Authored: Sat Nov 4 15:18:40 2017 -0700
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Sat Nov 4 18:56:36 2017 -0700
----------------------------------------------------------------------
.../sysml/utils/lite/BuildLiteExecution.java | 29 ++++++++++----------
.../functions/jmlc/BuildLiteJarTest.java | 6 +---
.../functions/jmlc/MulticlassSVMScoreTest.java | 6 +---
3 files changed, 17 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/aaa94818/src/main/java/org/apache/sysml/utils/lite/BuildLiteExecution.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/utils/lite/BuildLiteExecution.java b/src/main/java/org/apache/sysml/utils/lite/BuildLiteExecution.java
index 4ce6f33..d687a1d 100644
--- a/src/main/java/org/apache/sysml/utils/lite/BuildLiteExecution.java
+++ b/src/main/java/org/apache/sysml/utils/lite/BuildLiteExecution.java
@@ -46,8 +46,9 @@ import org.apache.sysml.runtime.util.DataConverter;
*/
public class BuildLiteExecution {
- protected static Logger log = Logger.getLogger(BuildLiteExecution.class);
-
+ private static Logger log = Logger.getLogger(BuildLiteExecution.class);
+ private static final String ROOT = "functions/jmlc/temp/";
+
public static void main(String[] args) throws Exception {
BasicConfigurator.configure();
@@ -94,11 +95,11 @@ public class BuildLiteExecution {
"write(predicted_y, \"./tmp\", format=\"text\");\n";
/* @formatter:on */
- File file = new File("temp/scoring-example.dml");
+ File file = new File(ROOT+"scoring-example.dml");
FileUtils.writeStringToFile(file, scriptString);
Connection conn = getConfiguredConnection();
- String dml = conn.readScript("temp/scoring-example.dml");
+ String dml = conn.readScript(ROOT+"scoring-example.dml");
PreparedScript script = conn.prepareScript(dml, new String[] { "W", "X" }, new String[] { "predicted_y" },
false);
@@ -158,17 +159,17 @@ public class BuildLiteExecution {
public static void jmlcWriteMatrix() throws Exception {
Connection conn = getConfiguredConnection();
PreparedScript script = conn.prepareScript(
- "x=matrix('1 2 3 4',rows=2,cols=2);write(x,'temp/x.csv',format='csv');", new String[] {},
+ "x=matrix('1 2 3 4',rows=2,cols=2);write(x,'"+ROOT+"x.csv',format='csv');", new String[] {},
new String[] {}, false);
script.executeScript();
/* @formatter:off */
String scriptString =
"m = matrix('1 2 3 0 0 0 7 8 9 0 0 0', rows=4, cols=3)\n" +
- "write(m, 'temp/m.txt', format='text')\n" +
- "write(m, 'temp/m.mm', format='mm')\n" +
- "write(m, 'temp/m.csv', format='csv')\n" +
- "write(m, 'temp/m.binary', format='binary')\n";
+ "write(m, '"+ROOT+"m.txt', format='text')\n" +
+ "write(m, '"+ROOT+"m.mm', format='mm')\n" +
+ "write(m, '"+ROOT+"m.csv', format='csv')\n" +
+ "write(m, '"+ROOT+"m.binary', format='binary')\n";
/* @formatter:on */
script = conn.prepareScript(scriptString, new String[] {}, new String[] {}, false);
@@ -179,12 +180,12 @@ public class BuildLiteExecution {
public static void jmlcReadMatrix() throws Exception {
Connection conn = getConfiguredConnection();
- PreparedScript script = conn.prepareScript("x=read('temp/x.csv',format='csv');y=x*2;print(toString(y));",
+ PreparedScript script = conn.prepareScript("x=read('"+ROOT+"x.csv',format='csv');y=x*2;print(toString(y));",
new String[] {}, new String[] {}, false);
script.executeScript();
/* @formatter:off */
- String scriptString = "m = read('temp/m.csv',format='csv')\n" +
+ String scriptString = "m = read('"+ROOT+"m.csv',format='csv')\n" +
"print(toString(m))\n" +
"print('min:' + min(m))\n" +
"print('max:' + max(m))\n" +
@@ -201,9 +202,9 @@ public class BuildLiteExecution {
// note: the following can be set to work using the following setting
// in the Connection class: cconf.set(ConfigType.IGNORE_READ_WRITE_METADATA, false);
- // "m2=read('temp/m.txt', format='text')\n" +
- // "m3=read('temp/m.mm', format='mm')\n" +
- // "m4=read('temp/m.binary', format='binary')\n" +
+ // "m2=read('"+ROOT+"m.txt', format='text')\n" +
+ // "m3=read('"+ROOT+"m.mm', format='mm')\n" +
+ // "m4=read('"+ROOT+"m.binary', format='binary')\n" +
// "print('m2:'+toString(m2))\n" +
// "print('m3:'+toString(m3))\n" +
// "print('m4:'+toString(m4))\n";
http://git-wip-us.apache.org/repos/asf/systemml/blob/aaa94818/src/test/java/org/apache/sysml/test/integration/functions/jmlc/BuildLiteJarTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/jmlc/BuildLiteJarTest.java b/src/test/java/org/apache/sysml/test/integration/functions/jmlc/BuildLiteJarTest.java
index 24c400f..620435e 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/jmlc/BuildLiteJarTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/jmlc/BuildLiteJarTest.java
@@ -46,12 +46,8 @@ public class BuildLiteJarTest extends AutomatedTestBase
}
@Test
- public void testJMLCWriteMatrix() throws Exception {
+ public void testJMLCWriteReadMatrix() throws Exception {
BuildLiteExecution.jmlcWriteMatrix();
- }
-
- @Test
- public void testJMLCReadMatrix() throws Exception {
BuildLiteExecution.jmlcReadMatrix();
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/aaa94818/src/test/java/org/apache/sysml/test/integration/functions/jmlc/MulticlassSVMScoreTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/jmlc/MulticlassSVMScoreTest.java b/src/test/java/org/apache/sysml/test/integration/functions/jmlc/MulticlassSVMScoreTest.java
index 9a0602a..eca6e52 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/jmlc/MulticlassSVMScoreTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/jmlc/MulticlassSVMScoreTest.java
@@ -181,12 +181,8 @@ public class MulticlassSVMScoreTest extends AutomatedTestBase
private ArrayList<double[][]> generateInputs( int num, int rows, int cols, double sparsity )
{
ArrayList<double[][]> ret = new ArrayList<double[][]>();
-
for( int i=0; i<num; i++ )
- {
- double[][] X = getRandomMatrix(rows, cols, -1, 1, sparsity, System.nanoTime());
- ret.add(X);
- }
+ ret.add(getRandomMatrix(rows, cols, -1, 1, sparsity, i));
return ret;
}
[2/3] systemml git commit: [SYSTEMML-1986] New nary cbind/rbind
(compiler/runtime for cp/spark)
Posted by mb...@apache.org.
http://git-wip-us.apache.org/repos/asf/systemml/blob/c0b6ef5c/src/test/java/org/apache/sysml/test/integration/functions/append/NaryCBindTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/append/NaryCBindTest.java b/src/test/java/org/apache/sysml/test/integration/functions/append/NaryCBindTest.java
new file mode 100644
index 0000000..13ba799
--- /dev/null
+++ b/src/test/java/org/apache/sysml/test/integration/functions/append/NaryCBindTest.java
@@ -0,0 +1,192 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.test.integration.functions.append;
+
+import java.util.HashMap;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
+
+public class NaryCBindTest extends AutomatedTestBase
+{
+ private final static String TEST_NAME = "NaryCbind";
+ private final static String TEST_DIR = "functions/append/";
+ private final static String TEST_CLASS_DIR = TEST_DIR + NaryCBindTest.class.getSimpleName() + "/";
+
+ private final static double epsilon=0.0000000001;
+
+ private final static int rows = 1101;
+ private final static int cols1 = 101;
+ private final static int cols2 = 79;
+ private final static int cols3 = 123;
+
+ private final static double sparsity1 = 0.7;
+ private final static double sparsity2 = 0.07;
+
+ @Override
+ public void setUp() {
+ TestUtils.clearAssertionInformation();
+ addTestConfiguration(TEST_NAME, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME, new String[] {"R"}));
+ }
+
+ @Test
+ public void testNaryCbindDenseDenseDenseCP() {
+ runCbindTest(false, false, false, ExecType.CP);
+ }
+
+ @Test
+ public void testNaryCbindDenseDenseSparseCP() {
+ runCbindTest(false, false, true, ExecType.CP);
+ }
+
+ @Test
+ public void testNaryCbindDenseSparseDenseCP() {
+ runCbindTest(false, true, false, ExecType.CP);
+ }
+
+ @Test
+ public void testNaryCbindDenseSparseSparseCP() {
+ runCbindTest(false, true, true, ExecType.CP);
+ }
+
+ @Test
+ public void testNaryCbindSparseDenseDenseCP() {
+ runCbindTest(true, false, false, ExecType.CP);
+ }
+
+ @Test
+ public void testNaryCbindSparseDenseSparseCP() {
+ runCbindTest(true, false, true, ExecType.CP);
+ }
+
+ @Test
+ public void testNaryCbindSparseSparseDenseCP() {
+ runCbindTest(true, true, false, ExecType.CP);
+ }
+
+ @Test
+ public void testNaryCbindSparseSparseSparseCP() {
+ runCbindTest(true, true, true, ExecType.CP);
+ }
+
+ @Test
+ public void testNaryCbindDenseDenseDenseSP() {
+ runCbindTest(false, false, false, ExecType.SPARK);
+ }
+
+ @Test
+ public void testNaryCbindDenseDenseSparseSP() {
+ runCbindTest(false, false, true, ExecType.SPARK);
+ }
+
+ @Test
+ public void testNaryCbindDenseSparseDenseSP() {
+ runCbindTest(false, true, false, ExecType.SPARK);
+ }
+
+ @Test
+ public void testNaryCbindDenseSparseSparseSP() {
+ runCbindTest(false, true, true, ExecType.SPARK);
+ }
+
+ @Test
+ public void testNaryCbindSparseDenseDenseSP() {
+ runCbindTest(true, false, false, ExecType.SPARK);
+ }
+
+ @Test
+ public void testNaryCbindSparseDenseSparseSP() {
+ runCbindTest(true, false, true, ExecType.SPARK);
+ }
+
+ @Test
+ public void testNaryCbindSparseSparseDenseSP() {
+ runCbindTest(true, true, false, ExecType.SPARK);
+ }
+
+ @Test
+ public void testNaryCbindSparseSparseSparseSP() {
+ runCbindTest(true, true, true, ExecType.CP);
+ }
+
+
+ public void runCbindTest(boolean sparse1, boolean sparse2, boolean sparse3, ExecType et)
+ {
+ RUNTIME_PLATFORM platformOld = rtplatform;
+ switch( et ) {
+ case MR: rtplatform = RUNTIME_PLATFORM.HADOOP; break;
+ case SPARK: rtplatform = RUNTIME_PLATFORM.SPARK; break;
+ default: rtplatform = RUNTIME_PLATFORM.HYBRID_SPARK; break;
+ }
+
+ boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
+ if( rtplatform == RUNTIME_PLATFORM.SPARK || rtplatform == RUNTIME_PLATFORM.HYBRID_SPARK )
+ DMLScript.USE_LOCAL_SPARK_CONFIG = true;
+
+ try
+ {
+ TestConfiguration config = getAndLoadTestConfiguration(TEST_NAME);
+ loadTestConfiguration(config);
+
+ String RI_HOME = SCRIPT_DIR + TEST_DIR;
+ fullDMLScriptName = RI_HOME + TEST_NAME + ".dml";
+ programArgs = new String[]{"-stats", "-args", input("A"),
+ input("B"), input("C"), output("R") };
+ fullRScriptName = RI_HOME + TEST_NAME + ".R";
+ rCmd = "Rscript" + " " + fullRScriptName + " " +
+ inputDir() + " "+ expectedDir();
+
+ //generate input data
+ double sp1 = sparse1 ? sparsity2 : sparsity1;
+ double sp2 = sparse2 ? sparsity2 : sparsity1;
+ double sp3 = sparse3 ? sparsity2 : sparsity1;
+ double[][] A = getRandomMatrix(rows, cols1, -1, 1, sp1, 711);
+ double[][] B = getRandomMatrix(rows, cols2, -1, 1, sp2, 722);
+ double[][] C = getRandomMatrix(rows, cols3, -1, 1, sp3, 733);
+ writeInputMatrixWithMTD("A", A, true);
+ writeInputMatrixWithMTD("B", B, true);
+ writeInputMatrixWithMTD("C", C, true);
+
+ //run tests
+ runTest(true, false, null, -1);
+ runRScript(true);
+
+ //compare result data
+ HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS("R");
+ HashMap<CellIndex, Double> rfile = readRMatrixFromFS("R");
+ TestUtils.compareMatrices(dmlfile, rfile, epsilon, "DML", "R");
+
+ //check for spark instructions
+ Assert.assertTrue(heavyHittersContainsSubString("sp_cbind")==(et==ExecType.SPARK));
+ }
+ finally {
+ rtplatform = platformOld;
+ DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/systemml/blob/c0b6ef5c/src/test/java/org/apache/sysml/test/integration/functions/append/NaryRBindTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/append/NaryRBindTest.java b/src/test/java/org/apache/sysml/test/integration/functions/append/NaryRBindTest.java
new file mode 100644
index 0000000..f820bf5
--- /dev/null
+++ b/src/test/java/org/apache/sysml/test/integration/functions/append/NaryRBindTest.java
@@ -0,0 +1,192 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.test.integration.functions.append;
+
+import java.util.HashMap;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
+
+public class NaryRBindTest extends AutomatedTestBase
+{
+ private final static String TEST_NAME = "NaryRbind";
+ private final static String TEST_DIR = "functions/append/";
+ private final static String TEST_CLASS_DIR = TEST_DIR + NaryRBindTest.class.getSimpleName() + "/";
+
+ private final static double epsilon=0.0000000001;
+
+ private final static int cols = 101;
+ private final static int rows1 = 1101;
+ private final static int rows2 = 1179;
+ private final static int rows3 = 1123;
+
+ private final static double sparsity1 = 0.7;
+ private final static double sparsity2 = 0.07;
+
+ @Override
+ public void setUp() {
+ TestUtils.clearAssertionInformation();
+ addTestConfiguration(TEST_NAME, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME, new String[] {"R"}));
+ }
+
+ @Test
+ public void testNaryRbindDenseDenseDenseCP() {
+ runRbindTest(false, false, false, ExecType.CP);
+ }
+
+ @Test
+ public void testNaryRbindDenseDenseSparseCP() {
+ runRbindTest(false, false, true, ExecType.CP);
+ }
+
+ @Test
+ public void testNaryRbindDenseSparseDenseCP() {
+ runRbindTest(false, true, false, ExecType.CP);
+ }
+
+ @Test
+ public void testNaryRbindDenseSparseSparseCP() {
+ runRbindTest(false, true, true, ExecType.CP);
+ }
+
+ @Test
+ public void testNaryRbindSparseDenseDenseCP() {
+ runRbindTest(true, false, false, ExecType.CP);
+ }
+
+ @Test
+ public void testNaryRbindSparseDenseSparseCP() {
+ runRbindTest(true, false, true, ExecType.CP);
+ }
+
+ @Test
+ public void testNaryRbindSparseSparseDenseCP() {
+ runRbindTest(true, true, false, ExecType.CP);
+ }
+
+ @Test
+ public void testNaryRbindSparseSparseSparseCP() {
+ runRbindTest(true, true, true, ExecType.CP);
+ }
+
+ @Test
+ public void testNaryRbindDenseDenseDenseSP() {
+ runRbindTest(false, false, false, ExecType.SPARK);
+ }
+
+ @Test
+ public void testNaryRbindDenseDenseSparseSP() {
+ runRbindTest(false, false, true, ExecType.SPARK);
+ }
+
+ @Test
+ public void testNaryRbindDenseSparseDenseSP() {
+ runRbindTest(false, true, false, ExecType.SPARK);
+ }
+
+ @Test
+ public void testNaryRbindDenseSparseSparseSP() {
+ runRbindTest(false, true, true, ExecType.SPARK);
+ }
+
+ @Test
+ public void testNaryRbindSparseDenseDenseSP() {
+ runRbindTest(true, false, false, ExecType.SPARK);
+ }
+
+ @Test
+ public void testNaryRbindSparseDenseSparseSP() {
+ runRbindTest(true, false, true, ExecType.SPARK);
+ }
+
+ @Test
+ public void testNaryRbindSparseSparseDenseSP() {
+ runRbindTest(true, true, false, ExecType.SPARK);
+ }
+
+ @Test
+ public void testNaryRbindSparseSparseSparseSP() {
+ runRbindTest(true, true, true, ExecType.CP);
+ }
+
+
+ public void runRbindTest(boolean sparse1, boolean sparse2, boolean sparse3, ExecType et)
+ {
+ RUNTIME_PLATFORM platformOld = rtplatform;
+ switch( et ) {
+ case MR: rtplatform = RUNTIME_PLATFORM.HADOOP; break;
+ case SPARK: rtplatform = RUNTIME_PLATFORM.SPARK; break;
+ default: rtplatform = RUNTIME_PLATFORM.HYBRID_SPARK; break;
+ }
+
+ boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
+ if( rtplatform == RUNTIME_PLATFORM.SPARK || rtplatform == RUNTIME_PLATFORM.HYBRID_SPARK )
+ DMLScript.USE_LOCAL_SPARK_CONFIG = true;
+
+ try
+ {
+ TestConfiguration config = getAndLoadTestConfiguration(TEST_NAME);
+ loadTestConfiguration(config);
+
+ String RI_HOME = SCRIPT_DIR + TEST_DIR;
+ fullDMLScriptName = RI_HOME + TEST_NAME + ".dml";
+ programArgs = new String[]{"-stats", "-args", input("A"),
+ input("B"), input("C"), output("R") };
+ fullRScriptName = RI_HOME + TEST_NAME + ".R";
+ rCmd = "Rscript" + " " + fullRScriptName + " " +
+ inputDir() + " "+ expectedDir();
+
+ //generate input data
+ double sp1 = sparse1 ? sparsity2 : sparsity1;
+ double sp2 = sparse2 ? sparsity2 : sparsity1;
+ double sp3 = sparse3 ? sparsity2 : sparsity1;
+ double[][] A = getRandomMatrix(rows1, cols, -1, 1, sp1, 711);
+ double[][] B = getRandomMatrix(rows2, cols, -1, 1, sp2, 722);
+ double[][] C = getRandomMatrix(rows3, cols, -1, 1, sp3, 733);
+ writeInputMatrixWithMTD("A", A, true);
+ writeInputMatrixWithMTD("B", B, true);
+ writeInputMatrixWithMTD("C", C, true);
+
+ //run tests
+ runTest(true, false, null, -1);
+ runRScript(true);
+
+ //compare result data
+ HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS("R");
+ HashMap<CellIndex, Double> rfile = readRMatrixFromFS("R");
+ TestUtils.compareMatrices(dmlfile, rfile, epsilon, "DML", "R");
+
+ //check for spark instructions
+ Assert.assertTrue(heavyHittersContainsSubString("sp_rbind")==(et==ExecType.SPARK));
+ }
+ finally {
+ rtplatform = platformOld;
+ DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/systemml/blob/c0b6ef5c/src/test/scripts/functions/append/NaryCbind.R
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/append/NaryCbind.R b/src/test/scripts/functions/append/NaryCbind.R
new file mode 100644
index 0000000..4cc8d89
--- /dev/null
+++ b/src/test/scripts/functions/append/NaryCbind.R
@@ -0,0 +1,32 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+args <- commandArgs(TRUE)
+options(digits=22)
+library("Matrix")
+
+A = as.matrix(readMM(paste(args[1], "A.mtx", sep="")))
+B = as.matrix(readMM(paste(args[1], "B.mtx", sep="")))
+C = as.matrix(readMM(paste(args[1], "C.mtx", sep="")))
+
+R = cbind(cbind(A, B), C);
+
+writeMM(as(R,"CsparseMatrix"), paste(args[2], "R", sep=""))
http://git-wip-us.apache.org/repos/asf/systemml/blob/c0b6ef5c/src/test/scripts/functions/append/NaryCbind.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/append/NaryCbind.dml b/src/test/scripts/functions/append/NaryCbind.dml
new file mode 100644
index 0000000..18331ad
--- /dev/null
+++ b/src/test/scripts/functions/append/NaryCbind.dml
@@ -0,0 +1,26 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+A = read($1)
+B = read($2)
+C = read($3)
+R = cbind(A, B, C);
+write(R, $4)
http://git-wip-us.apache.org/repos/asf/systemml/blob/c0b6ef5c/src/test/scripts/functions/append/NaryRbind.R
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/append/NaryRbind.R b/src/test/scripts/functions/append/NaryRbind.R
new file mode 100644
index 0000000..30ed40b
--- /dev/null
+++ b/src/test/scripts/functions/append/NaryRbind.R
@@ -0,0 +1,32 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+args <- commandArgs(TRUE)
+options(digits=22)
+library("Matrix")
+
+A = as.matrix(readMM(paste(args[1], "A.mtx", sep="")))
+B = as.matrix(readMM(paste(args[1], "B.mtx", sep="")))
+C = as.matrix(readMM(paste(args[1], "C.mtx", sep="")))
+
+R = rbind(rbind(A, B), C);
+
+writeMM(as(R,"CsparseMatrix"), paste(args[2], "R", sep=""))
http://git-wip-us.apache.org/repos/asf/systemml/blob/c0b6ef5c/src/test/scripts/functions/append/NaryRbind.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/append/NaryRbind.dml b/src/test/scripts/functions/append/NaryRbind.dml
new file mode 100644
index 0000000..515c4ce
--- /dev/null
+++ b/src/test/scripts/functions/append/NaryRbind.dml
@@ -0,0 +1,26 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+A = read($1)
+B = read($2)
+C = read($3)
+R = rbind(A, B, C);
+write(R, $4)
http://git-wip-us.apache.org/repos/asf/systemml/blob/c0b6ef5c/src/test_suites/java/org/apache/sysml/test/integration/functions/append/ZPackageSuite.java
----------------------------------------------------------------------
diff --git a/src/test_suites/java/org/apache/sysml/test/integration/functions/append/ZPackageSuite.java b/src/test_suites/java/org/apache/sysml/test/integration/functions/append/ZPackageSuite.java
index 5f0842b..df7cdd1 100644
--- a/src/test_suites/java/org/apache/sysml/test/integration/functions/append/ZPackageSuite.java
+++ b/src/test_suites/java/org/apache/sysml/test/integration/functions/append/ZPackageSuite.java
@@ -29,6 +29,8 @@ import org.junit.runners.Suite;
AppendChainTest.class,
AppendMatrixTest.class,
AppendVectorTest.class,
+ NaryCBindTest.class,
+ NaryRBindTest.class,
RBindCBindMatrixTest.class,
StringAppendTest.class,
})
[3/3] systemml git commit: [SYSTEMML-1986] New nary cbind/rbind
(compiler/runtime for cp/spark)
Posted by mb...@apache.org.
[SYSTEMML-1986] New nary cbind/rbind (compiler/runtime for cp/spark)
This patch generalizes the existing cbind and rbind from binary to nary
operations, which avoids unnecessary materialized intermediates and
simplifies its usage. Depending on the number of inputs, we now either
compile a BinaryOp or NaryOp which both however share a common block
runtime. For distributed operations, we support a new physical append
operator that relies on a simple "shift-pad-union-merge" approach and
thus, avoids unnecessary repeated shuffling.
Additionally, this patch also adds the related tests for rbind and cbind
with three inputs and all combinations of sparse and dense as well as
modifies some existing scripts to exploit the new capabilities.
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/c0b6ef5c
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/c0b6ef5c
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/c0b6ef5c
Branch: refs/heads/master
Commit: c0b6ef5ca92af4766ff7d758f9307ddbcbafc5b6
Parents: aaa9481
Author: Matthias Boehm <mb...@gmail.com>
Authored: Sat Nov 4 18:44:09 2017 -0700
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Sat Nov 4 18:56:37 2017 -0700
----------------------------------------------------------------------
scripts/algorithms/KM.dml | 8 +-
scripts/datagen/genRandData4StratStats.dml | 6 +-
scripts/nn/layers/lstm.dml | 2 +-
scripts/utils/image_utils.dml | 2 +-
src/main/java/org/apache/sysml/hops/Hop.java | 14 +-
.../java/org/apache/sysml/hops/MultipleOp.java | 185 ----------------
src/main/java/org/apache/sysml/hops/NaryOp.java | 219 +++++++++++++++++++
.../sysml/hops/rewrite/HopRewriteUtils.java | 17 ++
.../RewriteRemoveDanglingParentReferences.java | 6 +-
src/main/java/org/apache/sysml/lops/Lop.java | 4 +-
.../java/org/apache/sysml/lops/MultipleCP.java | 128 -----------
src/main/java/org/apache/sysml/lops/Nary.java | 128 +++++++++++
.../java/org/apache/sysml/lops/compile/Dag.java | 12 +-
.../sysml/parser/BuiltinFunctionExpression.java | 61 +++---
.../org/apache/sysml/parser/DMLTranslator.java | 33 +--
.../context/ExecutionContext.java | 14 +-
.../instructions/CPInstructionParser.java | 14 +-
.../instructions/SPInstructionParser.java | 8 +-
.../cp/BuiltinMultipleCPInstruction.java | 78 -------
.../runtime/instructions/cp/BuiltinNary.java | 74 +++++++
.../runtime/instructions/cp/CPInstruction.java | 4 +-
.../cp/MatrixBuiltinNaryCPInstruction.java | 55 +++++
.../cp/ScalarBuiltinMultipleCPInstruction.java | 101 ---------
.../cp/ScalarBuiltinNaryCPInstruction.java | 100 +++++++++
.../spark/BuiltinNarySPInstruction.java | 144 ++++++++++++
.../instructions/spark/SPInstruction.java | 2 +-
.../sysml/runtime/matrix/data/MatrixBlock.java | 70 +++---
.../functions/append/NaryCBindTest.java | 192 ++++++++++++++++
.../functions/append/NaryRBindTest.java | 192 ++++++++++++++++
src/test/scripts/functions/append/NaryCbind.R | 32 +++
src/test/scripts/functions/append/NaryCbind.dml | 26 +++
src/test/scripts/functions/append/NaryRbind.R | 32 +++
src/test/scripts/functions/append/NaryRbind.dml | 26 +++
.../functions/append/ZPackageSuite.java | 2 +
34 files changed, 1392 insertions(+), 599 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/c0b6ef5c/scripts/algorithms/KM.dml
----------------------------------------------------------------------
diff --git a/scripts/algorithms/KM.dml b/scripts/algorithms/KM.dml
index 26f42d6..8a4b1f7 100644
--- a/scripts/algorithms/KM.dml
+++ b/scripts/algorithms/KM.dml
@@ -167,7 +167,7 @@ if (GI_1_1 == 0 & SI_1_1 == 0) {
} else if (SI_1_1 == 0) {
Is = cbind (TE, GI);
} else {
- Is = cbind (TE, cbind (GI, SI));
+ Is = cbind (TE, GI, SI);
}
X = X %*% table (Is, seq (1, 2 + n_group_cols + n_stratum_cols), ncol (X), 2 + n_group_cols + n_stratum_cols);
@@ -201,7 +201,7 @@ if (n_group_cols > 0) {
Gi = matrix (1, rows = num_records, cols = 1);
}
if (n_stratum_cols > 0) {
- X = cbind (cbind (X[,1:2],Gi), X[,(3 + g):ncol (X)]);
+ X = cbind (X[,1:2], Gi, X[,(3 + g):ncol (X)]);
} else { # no strata
X = cbind (X[,1:2],Gi);
}
@@ -244,7 +244,7 @@ if (n_group_cols == 0 & n_stratum_cols == 0) {
X = cbind (X, matrix (1, rows = num_records, cols = 2));
SB = matrix (1, rows = 1, cols = 1);
} else if (n_group_cols == 0) {
- X = cbind (X[,1:2], cbind (matrix (1, rows = num_records, cols = 1), X[,3]));
+ X = cbind (X[,1:2], matrix (1, rows = num_records, cols = 1), X[,3]);
} else if (n_stratum_cols == 0) {
X = cbind (X, matrix (1, rows = num_records, cols = 1));
SB = matrix (1, rows = 1, cols = 1);
@@ -586,7 +586,7 @@ M = replace (target = M, pattern = "Infinity", replacement = "NaN");
# pull out non-empty rows from TEST
if (n_group_cols > 0 & n_stratum_cols > 0) {
- M = cbind (cbind (G_cols, S_cols), M);
+ M = cbind (G_cols, S_cols, M);
if (test_type != "none") {
TEST = cbind (G_cols_original, TEST);
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/c0b6ef5c/scripts/datagen/genRandData4StratStats.dml
----------------------------------------------------------------------
diff --git a/scripts/datagen/genRandData4StratStats.dml b/scripts/datagen/genRandData4StratStats.dml
index fb2aae0..6a4c07f 100644
--- a/scripts/datagen/genRandData4StratStats.dml
+++ b/scripts/datagen/genRandData4StratStats.dml
@@ -132,21 +132,21 @@ RY_records = Rand (rows = num_features, cols = num_records, pdf = "normal"); #
t_X = RX_records * stdev_X_within_strata + (RX_strata * stdev_X_between_strata + mean_X) %*% Smap;
t_Y = RY_records * stdev_Y_within_strata + (RY_strata * stdev_Y_between_strata + mean_Y) %*% Smap + (t_X * betas);
-Data = cbind (cbind (min_stratumID - 1 + SID, t(t_X)), t(t_Y));
+Data = cbind (min_stratumID - 1 + SID, t(t_X), t(t_Y));
# Set up the NaNs
RNaNS = Rand (rows = num_records, cols = 1, min = 1.0, max = 1.0, sparsity = prob_NaN_in_stratum);
RNaNX = Rand (rows = num_records, cols = num_features, min = 1.0, max = 1.0, sparsity = prob_NaN_in_X);
RNaNY = Rand (rows = num_records, cols = num_features, min = 1.0, max = 1.0, sparsity = prob_NaN_in_Y);
-Mask = (cbind (cbind (RNaNS, RNaNX), RNaNY)) != 0;
+Mask = cbind (RNaNS, RNaNX, RNaNY) != 0;
Data = Data + (1.0 - Mask) / (1.0 - Mask);
# Output the dataset and the auxiliaries
Xcid = t(seq (2, num_features + 1, 1));
Ycid = t(seq (num_features + 2, 2 * num_features + 1, 1));
-Aux = cbind (cbind (mean_X, mean_Y), betas);
+Aux = cbind (mean_X, mean_Y, betas);
write (Data, fileData, format=fmt);
write (Xcid, fileXcid, format=fmt);
http://git-wip-us.apache.org/repos/asf/systemml/blob/c0b6ef5c/scripts/nn/layers/lstm.dml
----------------------------------------------------------------------
diff --git a/scripts/nn/layers/lstm.dml b/scripts/nn/layers/lstm.dml
index 7c21f2b..4e8460c 100644
--- a/scripts/nn/layers/lstm.dml
+++ b/scripts/nn/layers/lstm.dml
@@ -206,7 +206,7 @@ backward = function(matrix[double] dout, matrix[double] dc,
df_raw = f * (1-f) * df
do_raw = o * (1-o) * do
dg_raw = (1-g^2) * dg
- difog_raw = cbind(di_raw, cbind(df_raw, cbind(do_raw, dg_raw))) # shape (N, 4M)
+ difog_raw = cbind(di_raw, df_raw, do_raw, dg_raw) # shape (N, 4M)
dW = dW + t(input) %*% difog_raw # shape (D+M, 4M)
db = db + colSums(difog_raw) # shape (1, 4M)
http://git-wip-us.apache.org/repos/asf/systemml/blob/c0b6ef5c/scripts/utils/image_utils.dml
----------------------------------------------------------------------
diff --git a/scripts/utils/image_utils.dml b/scripts/utils/image_utils.dml
index 813bc0e..3802f01 100644
--- a/scripts/utils/image_utils.dml
+++ b/scripts/utils/image_utils.dml
@@ -32,7 +32,7 @@ crop_rgb = function(matrix[double] input, int Hin, int Win, int Hout, int Wout)
temp_mask = matrix(1, rows=Hout, cols=Wout)
mask[start_h:end_h, start_w:end_w] = temp_mask
mask = matrix(mask, rows=1, cols=Hin*Win)
- mask = cbind(cbind(mask, mask), mask)
+ mask = cbind(mask, mask, mask)
out = removeEmpty(target=(input+1), margin="cols", select=mask) - 1
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/c0b6ef5c/src/main/java/org/apache/sysml/hops/Hop.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/Hop.java b/src/main/java/org/apache/sysml/hops/Hop.java
index ec0a80c..350c64a 100644
--- a/src/main/java/org/apache/sysml/hops/Hop.java
+++ b/src/main/java/org/apache/sysml/hops/Hop.java
@@ -38,7 +38,7 @@ import org.apache.sysml.lops.Data;
import org.apache.sysml.lops.Lop;
import org.apache.sysml.lops.LopProperties.ExecType;
import org.apache.sysml.lops.LopsException;
-import org.apache.sysml.lops.MultipleCP;
+import org.apache.sysml.lops.Nary;
import org.apache.sysml.lops.ReBlock;
import org.apache.sysml.lops.UnaryCP;
import org.apache.sysml.parser.Expression.DataType;
@@ -1100,8 +1100,8 @@ public abstract class Hop implements ParseInfo
}
// Operations that require a variable number of operands
- public enum MultiInputOp {
- PRINTF
+ public enum OpOpN {
+ PRINTF, CBIND, RBIND,
}
public enum AggOp {
@@ -1351,10 +1351,12 @@ public abstract class Hop implements ParseInfo
* constructLops() method that is used to construct the Lops that correspond
* to a Hop.
*/
- protected static final HashMap<MultiInputOp, MultipleCP.OperationType> MultipleOperandOperationHopTypeToLopType;
+ protected static final HashMap<OpOpN, Nary.OperationType> HopsOpOpNLops;
static {
- MultipleOperandOperationHopTypeToLopType = new HashMap<>();
- MultipleOperandOperationHopTypeToLopType.put(MultiInputOp.PRINTF, MultipleCP.OperationType.PRINTF);
+ HopsOpOpNLops = new HashMap<>();
+ HopsOpOpNLops.put(OpOpN.PRINTF, Nary.OperationType.PRINTF);
+ HopsOpOpNLops.put(OpOpN.CBIND, Nary.OperationType.CBIND);
+ HopsOpOpNLops.put(OpOpN.RBIND, Nary.OperationType.RBIND);
}
protected static final HashMap<Hop.OpOp1, String> HopsOpOp12String;
http://git-wip-us.apache.org/repos/asf/systemml/blob/c0b6ef5c/src/main/java/org/apache/sysml/hops/MultipleOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/MultipleOp.java b/src/main/java/org/apache/sysml/hops/MultipleOp.java
deleted file mode 100644
index 5c178c0..0000000
--- a/src/main/java/org/apache/sysml/hops/MultipleOp.java
+++ /dev/null
@@ -1,185 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.sysml.hops;
-
-import java.util.ArrayList;
-
-import org.apache.sysml.lops.Lop;
-import org.apache.sysml.lops.LopProperties.ExecType;
-import org.apache.sysml.lops.LopsException;
-import org.apache.sysml.lops.MultipleCP;
-import org.apache.sysml.parser.Expression.DataType;
-import org.apache.sysml.parser.Expression.ValueType;
-
-/**
- * The MultipleOp Hop allows for a variable number of operands. Functionality
- * such as 'printf' (overloaded into the existing print function) is an example
- * of an operation that potentially takes a variable number of operands.
- *
- */
-public class MultipleOp extends Hop {
- protected MultiInputOp _op = null;
-
- protected MultipleOp() {
- }
-
- /**
- * MultipleOp constructor.
- *
- * @param name
- * the target name, typically set by the DMLTranslator when
- * constructing Hops. (For example, 'parsertemp1'.)
- * @param dataType
- * the target data type (SCALAR for printf)
- * @param valueType
- * the target value type (STRING for printf)
- * @param multipleOperandOperation
- * the operation type (such as PRINTF)
- * @param inputs
- * a variable number of input Hops
- * @throws HopsException
- * thrown if a HopsException occurs
- */
- public MultipleOp(String name, DataType dataType, ValueType valueType,
- MultiInputOp multipleOperandOperation, Hop... inputs) throws HopsException {
- super(name, dataType, valueType);
- _op = multipleOperandOperation;
-
- for (int i = 0; i < inputs.length; i++) {
- getInput().add(i, inputs[i]);
- inputs[i].getParent().add(this);
- }
- }
-
- /** MultipleOp may have any number of inputs. */
- @Override
- public void checkArity() throws HopsException {}
-
- public MultiInputOp getOp() {
- return _op;
- }
-
- @Override
- public String getOpString() {
- return "m(" + _op.name().toLowerCase() + ")";
- }
-
- @Override
- public boolean isGPUEnabled() {
- return false;
- }
-
- /**
- * Construct the corresponding Lops for this Hop
- */
- @Override
- public Lop constructLops() throws HopsException, LopsException {
- // reuse existing lop
- if (getLops() != null)
- return getLops();
-
- try {
- ArrayList<Hop> inHops = getInput();
- Lop[] inLops = new Lop[inHops.size()];
- for (int i = 0; i < inHops.size(); i++) {
- Hop inHop = inHops.get(i);
- Lop inLop = inHop.constructLops();
- inLops[i] = inLop;
- }
-
- MultipleCP.OperationType opType = MultipleOperandOperationHopTypeToLopType.get(_op);
- if (opType == null) {
- throw new HopsException("Unknown MultipleCP Lop operation type for "
- + "MultipleOperandOperation Hop type '" + _op + "'");
- }
-
- MultipleCP multipleCPLop = new MultipleCP(opType, getDataType(), getValueType(), inLops);
- setOutputDimensions(multipleCPLop);
- setLineNumbers(multipleCPLop);
- setLops(multipleCPLop);
- } catch (Exception e) {
- throw new HopsException(this.printErrorLocation() + "error constructing Lops for MultipleOp Hop -- \n ", e);
- }
-
- // add reblock/checkpoint lops if necessary
- constructAndSetLopsDataFlowProperties();
-
- return getLops();
- }
-
- @Override
- protected double computeOutputMemEstimate(long dim1, long dim2, long nnz) {
- double sparsity = OptimizerUtils.getSparsity(dim1, dim2, nnz);
- return OptimizerUtils.estimateSizeExactSparsity(dim1, dim2, sparsity);
- }
-
- @Override
- public boolean allowsAllExecTypes() {
- return false;
- }
-
- @Override
- protected ExecType optFindExecType() throws HopsException {
- return ExecType.CP;
- }
-
- @Override
- public void refreshSizeInformation() {
- // do nothing
- }
-
- @Override
- public Object clone() throws CloneNotSupportedException {
- MultipleOp multipleOp = new MultipleOp();
-
- // copy generic attributes
- multipleOp.clone(this, false);
-
- // copy specific attributes
- multipleOp._op = _op;
-
- return multipleOp;
- }
-
- @Override
- public boolean compare(Hop that) {
- if (!(that instanceof MultipleOp))
- return false;
-
- if (_op == MultiInputOp.PRINTF) {
- return false;
- }
-
- // if add new multiple operand types in addition to PRINTF,
- // probably need to modify this.
- MultipleOp mo = (MultipleOp) that;
- return (_op == mo._op);
- }
-
- @Override
- protected double computeIntermediateMemEstimate(long dim1, long dim2, long nnz) {
- return 0;
- }
-
- @Override
- protected long[] inferOutputCharacteristics(MemoTable memo) {
- return null;
- }
-}
http://git-wip-us.apache.org/repos/asf/systemml/blob/c0b6ef5c/src/main/java/org/apache/sysml/hops/NaryOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/NaryOp.java b/src/main/java/org/apache/sysml/hops/NaryOp.java
new file mode 100644
index 0000000..cc90044
--- /dev/null
+++ b/src/main/java/org/apache/sysml/hops/NaryOp.java
@@ -0,0 +1,219 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.hops;
+
+import org.apache.sysml.hops.rewrite.HopRewriteUtils;
+import org.apache.sysml.lops.Lop;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.lops.LopsException;
+import org.apache.sysml.lops.Nary;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
+
+/**
+ * The NaryOp Hop allows for a variable number of operands. Functionality
+ * such as 'printf' (overloaded into the existing print function) is an example
+ * of an operation that potentially takes a variable number of operands.
+ *
+ */
+public class NaryOp extends Hop {
+ protected OpOpN _op = null;
+
+ protected NaryOp() {
+ }
+
+ /**
+ * NaryOp constructor.
+ *
+ * @param name
+ * the target name, typically set by the DMLTranslator when
+ * constructing Hops. (For example, 'parsertemp1'.)
+ * @param dataType
+ * the target data type (SCALAR for printf)
+ * @param valueType
+ * the target value type (STRING for printf)
+ * @param multipleOperandOperation
+ * the operation type (such as PRINTF)
+ * @param inputs
+ * a variable number of input Hops
+ * @throws HopsException
+ * thrown if a HopsException occurs
+ */
+ public NaryOp(String name, DataType dataType, ValueType valueType,
+ OpOpN op, Hop... inputs) throws HopsException {
+ super(name, dataType, valueType);
+ _op = op;
+
+ for (int i = 0; i < inputs.length; i++) {
+ getInput().add(i, inputs[i]);
+ inputs[i].getParent().add(this);
+ }
+ }
+
+ /** MultipleOp may have any number of inputs. */
+ @Override
+ public void checkArity() throws HopsException {}
+
+ public OpOpN getOp() {
+ return _op;
+ }
+
+ @Override
+ public String getOpString() {
+ return "m(" + _op.name().toLowerCase() + ")";
+ }
+
+ @Override
+ public boolean isGPUEnabled() {
+ return false;
+ }
+
+ /**
+ * Construct the corresponding Lops for this Hop
+ */
+ @Override
+ public Lop constructLops() throws HopsException, LopsException {
+ // reuse existing lop
+ if (getLops() != null)
+ return getLops();
+
+ try {
+ Lop[] inLops = new Lop[getInput().size()];
+ for (int i = 0; i < getInput().size(); i++)
+ inLops[i] = getInput().get(i).constructLops();
+
+ Nary.OperationType opType = HopsOpOpNLops.get(_op);
+ if (opType == null)
+ throw new HopsException("Unknown Nary Lop type for '"+_op+"'");
+
+ ExecType et = optFindExecType();
+ Nary multipleCPLop = new Nary(opType, getDataType(), getValueType(), inLops, et);
+ setOutputDimensions(multipleCPLop);
+ setLineNumbers(multipleCPLop);
+ setLops(multipleCPLop);
+ }
+ catch (Exception e) {
+ throw new HopsException(this.printErrorLocation() + "error constructing Lops for NaryOp -- \n ", e);
+ }
+
+ // add reblock/checkpoint lops if necessary
+ constructAndSetLopsDataFlowProperties();
+
+ return getLops();
+ }
+
+ @Override
+ public boolean allowsAllExecTypes() {
+ return false;
+ }
+
+ @Override
+ protected double computeOutputMemEstimate(long dim1, long dim2, long nnz) {
+ double sparsity = OptimizerUtils.getSparsity(dim1, dim2, nnz);
+ return OptimizerUtils.estimateSizeExactSparsity(dim1, dim2, sparsity);
+ }
+
+ @Override
+ protected ExecType optFindExecType() throws HopsException {
+
+ checkAndSetForcedPlatform();
+
+ ExecType REMOTE = OptimizerUtils.isSparkExecutionMode() ? ExecType.SPARK : ExecType.MR;
+
+ //forced / memory-based / threshold-based decision
+ if( _etypeForced != null ) {
+ _etype = _etypeForced;
+ }
+ else
+ {
+ if ( OptimizerUtils.isMemoryBasedOptLevel() )
+ _etype = findExecTypeByMemEstimate();
+ // Choose CP, if the input dimensions are below threshold or if the input is a vector
+ else if ( areDimsBelowThreshold() )
+ _etype = ExecType.CP;
+ else
+ _etype = REMOTE;
+
+ //check for valid CP dimensions and matrix size
+ checkAndSetInvalidCPDimsAndSize();
+ }
+
+ //mark for recompile (forever)
+ setRequiresRecompileIfNecessary();
+
+ //ensure cp exec type for single-node operations
+ if ( _op == OpOpN.PRINTF )
+ _etype = ExecType.CP;
+
+ return _etype;
+ }
+
+ @Override
+ protected double computeIntermediateMemEstimate(long dim1, long dim2, long nnz) {
+ return 0;
+ }
+
+ @Override
+ protected long[] inferOutputCharacteristics(MemoTable memo) {
+ return null; //do nothing
+ }
+
+ @Override
+ public void refreshSizeInformation() {
+ switch( _op ) {
+ case CBIND:
+ setDim1(HopRewriteUtils.getMaxInputDim(this, true));
+ setDim2(HopRewriteUtils.getSumValidInputDims(this, false));
+ break;
+ case RBIND:
+ setDim1(HopRewriteUtils.getSumValidInputDims(this, false));
+ setDim2(HopRewriteUtils.getMaxInputDim(this, true));
+ break;
+ case PRINTF:
+ //do nothing:
+ }
+ }
+
+ @Override
+ public Object clone() throws CloneNotSupportedException {
+ NaryOp multipleOp = new NaryOp();
+
+ // copy generic attributes
+ multipleOp.clone(this, false);
+
+ // copy specific attributes
+ multipleOp._op = _op;
+
+ return multipleOp;
+ }
+
+ @Override
+ public boolean compare(Hop that) {
+ if (!(that instanceof NaryOp) || _op == OpOpN.PRINTF)
+ return false;
+
+ NaryOp that2 = (NaryOp) that;
+ boolean ret = (_op == that2._op
+ && getInput().size() == that2.getInput().size());
+ for( int i=0; i<getInput().size() && ret; i++ )
+ ret &= (getInput().get(i) == that2.getInput().get(i));
+ return ret;
+ }
+}
http://git-wip-us.apache.org/repos/asf/systemml/blob/c0b6ef5c/src/main/java/org/apache/sysml/hops/rewrite/HopRewriteUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/rewrite/HopRewriteUtils.java b/src/main/java/org/apache/sysml/hops/rewrite/HopRewriteUtils.java
index ad2392a..b6db466 100644
--- a/src/main/java/org/apache/sysml/hops/rewrite/HopRewriteUtils.java
+++ b/src/main/java/org/apache/sysml/hops/rewrite/HopRewriteUtils.java
@@ -1214,4 +1214,21 @@ public class HopRewriteUtils
|| sb instanceof IfStatementBlock
|| sb instanceof ForStatementBlock); //incl parfor
}
+
+ public static long getMaxInputDim(Hop hop, boolean dim1) {
+ return hop.getInput().stream().mapToLong(
+ h -> (dim1?h.getDim1():h.getDim2())).max().orElse(-1);
+ }
+
+ public static long getSumValidInputDims(Hop hop, boolean dim1) {
+ if( !hasValidInputDims(hop, dim1) )
+ return -1;
+ return hop.getInput().stream().mapToLong(
+ h -> (dim1?h.getDim1():h.getDim2())).sum();
+ }
+
+ public static boolean hasValidInputDims(Hop hop, boolean dim1) {
+ return hop.getInput().stream().allMatch(
+ h -> (dim1?h.getDim1()>0:h.getDim2()>0));
+ }
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/c0b6ef5c/src/main/java/org/apache/sysml/hops/rewrite/RewriteRemoveDanglingParentReferences.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/rewrite/RewriteRemoveDanglingParentReferences.java b/src/main/java/org/apache/sysml/hops/rewrite/RewriteRemoveDanglingParentReferences.java
index ac8fb68..573b9fb 100644
--- a/src/main/java/org/apache/sysml/hops/rewrite/RewriteRemoveDanglingParentReferences.java
+++ b/src/main/java/org/apache/sysml/hops/rewrite/RewriteRemoveDanglingParentReferences.java
@@ -25,10 +25,10 @@ import org.apache.sysml.hops.DataOp;
import org.apache.sysml.hops.FunctionOp;
import org.apache.sysml.hops.Hop;
import org.apache.sysml.hops.Hop.DataOpTypes;
-import org.apache.sysml.hops.Hop.MultiInputOp;
+import org.apache.sysml.hops.Hop.OpOpN;
import org.apache.sysml.hops.Hop.OpOp1;
import org.apache.sysml.hops.HopsException;
-import org.apache.sysml.hops.MultipleOp;
+import org.apache.sysml.hops.NaryOp;
import org.apache.sysml.hops.UnaryOp;
/**
@@ -108,7 +108,7 @@ public class RewriteRemoveDanglingParentReferences extends HopRewriteRule
return (hop instanceof DataOp && ((DataOp)hop).isWrite())
|| (hop instanceof UnaryOp && ((UnaryOp)hop).getOp()==OpOp1.STOP)
|| (hop instanceof UnaryOp && ((UnaryOp)hop).getOp()==OpOp1.PRINT)
- || (hop instanceof MultipleOp && ((MultipleOp)hop).getOp()==MultiInputOp.PRINTF)
+ || (hop instanceof NaryOp && ((NaryOp)hop).getOp()==OpOpN.PRINTF)
|| (hop instanceof FunctionOp)
|| (hop instanceof DataOp && ((DataOp)hop).getDataOpType()==DataOpTypes.FUNCTIONOUTPUT);
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/c0b6ef5c/src/main/java/org/apache/sysml/lops/Lop.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/Lop.java b/src/main/java/org/apache/sysml/lops/Lop.java
index 499cd59..8909ea4 100644
--- a/src/main/java/org/apache/sysml/lops/Lop.java
+++ b/src/main/java/org/apache/sysml/lops/Lop.java
@@ -41,7 +41,7 @@ public abstract class Lop
Data, DataGen, //CP/MR read/write/datagen
ReBlock, CSVReBlock, //MR reblock operations
MMCJ, MMRJ, MMTSJ, PMMJ, MapMult, MapMultChain, //MR matrix multiplications
- UnaryCP, UNARY, BinaryCP, Binary, Ternary, //CP/MR unary/binary/ternary
+ UnaryCP, UNARY, BinaryCP, Binary, Ternary, Nary, //CP/MR unary/binary/ternary
RightIndex, LeftIndex, ZeroOut, //CP/MR indexing
Aggregate, PartialAggregate, //CP/MR aggregation
BinUaggChain, UaggOuterChain, //CP/MR aggregation
@@ -59,8 +59,6 @@ public abstract class Lop
Checkpoint, //Spark persist into storage level
PlusMult, MinusMult, //CP
SpoofFused, //CP/SP generated fused operator
- /** CP operation on a variable number of operands */
- MULTIPLE_CP
}
/**
http://git-wip-us.apache.org/repos/asf/systemml/blob/c0b6ef5c/src/main/java/org/apache/sysml/lops/MultipleCP.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/MultipleCP.java b/src/main/java/org/apache/sysml/lops/MultipleCP.java
deleted file mode 100644
index 2560861..0000000
--- a/src/main/java/org/apache/sysml/lops/MultipleCP.java
+++ /dev/null
@@ -1,128 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.sysml.lops;
-
-import org.apache.sysml.lops.LopProperties.ExecLocation;
-import org.apache.sysml.lops.LopProperties.ExecType;
-import org.apache.sysml.lops.compile.JobType;
-import org.apache.sysml.parser.Expression.DataType;
-import org.apache.sysml.parser.Expression.ValueType;
-
-/**
- * Lop to perform an operation on a variable number of operands.
- *
- */
-public class MultipleCP extends Lop {
-
- public enum OperationType {
- PRINTF
- }
-
- OperationType operationType;
-
- public MultipleCP(OperationType operationType, DataType dt, ValueType vt, Lop... inputLops) {
- super(Lop.Type.MULTIPLE_CP, dt, vt);
- this.operationType = operationType;
- for (Lop inputLop : inputLops) {
- addInput(inputLop);
- inputLop.addOutput(this);
- }
-
- boolean breaksAlignment = false; // ?
- boolean aligner = false; // ?
- boolean definesMRJob = false; // ?
- lps.addCompatibility(JobType.INVALID); // ?
- this.lps.setProperties(inputs, ExecType.CP, ExecLocation.ControlProgram, breaksAlignment, aligner,
- definesMRJob); // ?
- }
-
- @Override
- public String toString() {
- return "Operation Type: " + operationType;
- }
-
- public OperationType getOperationType() {
- return operationType;
- }
-
- /**
- * Generate the complete instruction string for this Lop. This instruction
- * string can have a variable number of input operands. It displays the
- * following:
- *
- * <ul>
- * <li>Execution type (CP, SPARK, etc.)
- * <li>Operand delimiter (°)
- * <li>Opcode (printf, etc.)
- * <li>Operand delimiter (°)
- * <li>Variable number of inputs, each followed by an operand delimiter
- * (°)
- * <ul>
- * <li>Input consists of (label · data type · value type
- * · is literal)
- * </ul>
- * <li>Output consisting of (label · data type · value
- * type)
- * </ul>
- *
- * Example: <br>
- * The following DML<br>
- * <code>print('hello %s', 'world')</code><br>
- * generates the instruction string:<br>
- * <code>CP°printf°hello %s·SCALAR·STRING·true°world·SCALAR·STRING·true°_Var1·SCALAR·STRING</code><br>
- *
- * Note: This generated instruction string is parsed in the
- * parseInstruction() method of BuiltinMultipleCPInstruction, which parses
- * the instruction string to generate an instruction object that is a
- * subclass of BuiltinMultipleCPInstruction.
- */
- @Override
- public String getInstructions(String output) throws LopsException {
- String opString = getOpcode();
-
- StringBuilder sb = new StringBuilder();
-
- sb.append(getExecType());
- sb.append(Lop.OPERAND_DELIMITOR);
-
- sb.append(opString);
- sb.append(OPERAND_DELIMITOR);
-
- for (Lop input : inputs) {
- sb.append(input.prepScalarInputOperand(getExecType()));
- sb.append(OPERAND_DELIMITOR);
- }
-
- sb.append(prepOutputOperand(output));
-
- return sb.toString();
- }
-
- private String getOpcode() throws LopsException {
- switch (operationType) {
- case PRINTF:
- return OperationType.PRINTF.toString().toLowerCase();
- default:
- throw new UnsupportedOperationException(
- "MultipleCP operation type (" + operationType + ") is not defined.");
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/systemml/blob/c0b6ef5c/src/main/java/org/apache/sysml/lops/Nary.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/Nary.java b/src/main/java/org/apache/sysml/lops/Nary.java
new file mode 100644
index 0000000..90966fa
--- /dev/null
+++ b/src/main/java/org/apache/sysml/lops/Nary.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.lops;
+
+import org.apache.sysml.lops.LopProperties.ExecLocation;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.lops.compile.JobType;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
+
+/**
+ * Lop to perform an operation on a variable number of operands.
+ *
+ */
+public class Nary extends Lop {
+
+ public enum OperationType {
+ PRINTF, CBIND, RBIND,
+ }
+
+ private OperationType operationType;
+
+ public Nary(OperationType operationType, DataType dt, ValueType vt, Lop[] inputLops, ExecType et)
+ throws LopsException
+ {
+ super(Lop.Type.Nary, dt, vt);
+ this.operationType = operationType;
+ for (Lop inputLop : inputLops) {
+ addInput(inputLop);
+ inputLop.addOutput(this);
+ }
+
+ if( et == ExecType.CP || et == ExecType.SPARK ) {
+ lps.addCompatibility(JobType.INVALID);
+ lps.setProperties(inputs, et, ExecLocation.ControlProgram, false, false, false);
+ }
+ else {
+ throw new LopsException("Unsupported exec type for nary lop:" + et.name());
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "Operation Type: " + operationType;
+ }
+
+ public OperationType getOperationType() {
+ return operationType;
+ }
+
+ /**
+ * Generate the complete instruction string for this Lop. This instruction
+ * string can have a variable number of input operands. It displays the
+ * following:
+ *
+ * <ul>
+ * <li>Execution type (CP, SPARK, etc.)
+ * <li>Operand delimiter (°)
+ * <li>Opcode (printf, etc.)
+ * <li>Operand delimiter (°)
+ * <li>Variable number of inputs, each followed by an operand delimiter
+ * (°)
+ * <ul>
+ * <li>Input consists of (label · data type · value type
+ * · is literal)
+ * </ul>
+ * <li>Output consisting of (label · data type · value
+ * type)
+ * </ul>
+ *
+ * Example: <br>
+ * The following DML<br>
+ * <code>print('hello %s', 'world')</code><br>
+ * generates the instruction string:<br>
+ * <code>CP°printf°hello %s·SCALAR·STRING·true°world·SCALAR·STRING·true°_Var1·SCALAR·STRING</code><br>
+ *
+ */
+ @Override
+ public String getInstructions(String[] inputs, String output) throws LopsException {
+ String opString = getOpcode();
+
+ StringBuilder sb = new StringBuilder();
+
+ sb.append(getExecType());
+ sb.append(Lop.OPERAND_DELIMITOR);
+
+ sb.append(opString);
+ sb.append(OPERAND_DELIMITOR);
+
+ for( int i=0; i<inputs.length; i++ ) {
+ sb.append(getInputs().get(i).prepInputOperand(inputs[i]));
+ sb.append(OPERAND_DELIMITOR);
+ }
+
+ sb.append(prepOutputOperand(output));
+
+ return sb.toString();
+ }
+
+ private String getOpcode() throws LopsException {
+ switch (operationType) {
+ case PRINTF:
+ case CBIND:
+ case RBIND:
+ return operationType.name().toLowerCase();
+ default:
+ throw new UnsupportedOperationException(
+ "Nary operation type (" + operationType + ") is not defined.");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/systemml/blob/c0b6ef5c/src/main/java/org/apache/sysml/lops/compile/Dag.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/compile/Dag.java b/src/main/java/org/apache/sysml/lops/compile/Dag.java
index 91f49f0..4edf230 100644
--- a/src/main/java/org/apache/sysml/lops/compile/Dag.java
+++ b/src/main/java/org/apache/sysml/lops/compile/Dag.java
@@ -1383,14 +1383,16 @@ public class Dag<N extends Lop>
inputs[count++] = in.getOutputParameters().getLabel();
count = 0;
for( Lop out : node.getOutputs() )
- {
outputs[count++] = out.getOutputParameters().getLabel();
- }
-
inst_string = node.getInstructions(inputs, outputs);
}
- else if (node.getType() == Lop.Type.MULTIPLE_CP) { // ie, MultipleCP class
- inst_string = node.getInstructions(node.getOutputParameters().getLabel());
+ else if (node.getType() == Lop.Type.Nary) {
+ String[] inputs = new String[node.getInputs().size()];
+ int count = 0;
+ for( Lop in : node.getInputs() )
+ inputs[count++] = in.getOutputParameters().getLabel();
+ inst_string = node.getInstructions(inputs,
+ node.getOutputParameters().getLabel());
}
else {
if ( node.getInputs().isEmpty() ) {
http://git-wip-us.apache.org/repos/asf/systemml/blob/c0b6ef5c/src/main/java/org/apache/sysml/parser/BuiltinFunctionExpression.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/BuiltinFunctionExpression.java b/src/main/java/org/apache/sysml/parser/BuiltinFunctionExpression.java
index 9a8129c..566ddf8 100644
--- a/src/main/java/org/apache/sysml/parser/BuiltinFunctionExpression.java
+++ b/src/main/java/org/apache/sysml/parser/BuiltinFunctionExpression.java
@@ -91,6 +91,10 @@ public class BuiltinFunctionExpression extends DataIdentifier
return _args;
}
+ public Expression getExpr(int i) {
+ return (_args.length > i ? _args[i] : null);
+ }
+
@Override
public void validateExpression(MultiAssignmentStatement stmt, HashMap<String, DataIdentifier> ids, HashMap<String, ConstIdentifier> constVars, boolean conditional)
throws LanguageException
@@ -539,50 +543,53 @@ public class BuiltinFunctionExpression extends DataIdentifier
break;
case CBIND:
- case RBIND:
- checkNumParameters(2);
-
+ case RBIND:
//scalar string append (string concatenation with \n)
if( getFirstExpr().getOutput().getDataType()==DataType.SCALAR ) {
+ checkNumParameters(2);
checkScalarParam(getFirstExpr());
checkScalarParam(getSecondExpr());
checkValueTypeParam(getFirstExpr(), ValueType.STRING);
checkValueTypeParam(getSecondExpr(), ValueType.STRING);
}
//matrix append (rbind/cbind)
- else {
- checkMatrixFrameParam(getFirstExpr());
- checkMatrixFrameParam(getSecondExpr());
+ else {
+ if( getAllExpr().length < 2 )
+ raiseValidateError("Invalid number of arguments for "+getOpCode(), conditional);
+ for(int i=0; i<getAllExpr().length; i++)
+ checkMatrixFrameParam(getExpr(i));
}
output.setDataType(id.getDataType());
output.setValueType(id.getValueType());
// set output dimensions and validate consistency
- long appendDim1 = -1, appendDim2 = -1;
long m1rlen = getFirstExpr().getOutput().getDim1();
long m1clen = getFirstExpr().getOutput().getDim2();
- long m2rlen = getSecondExpr().getOutput().getDim1();
- long m2clen = getSecondExpr().getOutput().getDim2();
-
- if( getOpCode() == BuiltinFunctionOp.CBIND ) {
- if (m1rlen > 0 && m2rlen > 0 && m1rlen!=m2rlen) {
- raiseValidateError("inputs to cbind must have same number of rows: input 1 rows: " +
- m1rlen+", input 2 rows: "+m2rlen, conditional, LanguageErrorCodes.INVALID_PARAMETERS);
- }
- appendDim1 = (m1rlen>0) ? m1rlen : m2rlen;
- appendDim2 = (m1clen>0 && m2clen>0)? m1clen + m2clen : -1;
- }
- else if( getOpCode() == BuiltinFunctionOp.RBIND ) {
- if (m1clen > 0 && m2clen > 0 && m1clen!=m2clen) {
- raiseValidateError("inputs to rbind must have same number of columns: input 1 columns: " +
- m1clen+", input 2 columns: "+m2clen, conditional, LanguageErrorCodes.INVALID_PARAMETERS);
- }
- appendDim1 = (m1rlen>0 && m2rlen>0)? m1rlen + m2rlen : -1;
- appendDim2 = (m1clen>0) ? m1clen : m2clen;
- }
+ long appendDim1 = m1rlen, appendDim2 = m1clen;
- output.setDimensions(appendDim1, appendDim2);
+ for(int i=1; i<getAllExpr().length; i++) {
+ long m2rlen = getExpr(i).getOutput().getDim1();
+ long m2clen = getExpr(i).getOutput().getDim2();
+
+ if( getOpCode() == BuiltinFunctionOp.CBIND ) {
+ if (m1rlen > 0 && m2rlen > 0 && m1rlen!=m2rlen) {
+ raiseValidateError("inputs to cbind must have same number of rows: input 1 rows: " +
+ m1rlen+", input 2 rows: "+m2rlen, conditional, LanguageErrorCodes.INVALID_PARAMETERS);
+ }
+ appendDim1 = (m2rlen>0) ? m2rlen : appendDim1;
+ appendDim2 = (appendDim2>0 && m2clen>0) ? appendDim2 + m2clen : -1;
+ }
+ else if( getOpCode() == BuiltinFunctionOp.RBIND ) {
+ if (m1clen > 0 && m2clen > 0 && m1clen!=m2clen) {
+ raiseValidateError("inputs to rbind must have same number of columns: input 1 columns: " +
+ m1clen+", input 2 columns: "+m2clen, conditional, LanguageErrorCodes.INVALID_PARAMETERS);
+ }
+ appendDim1 = (appendDim1>0 && m2rlen>0)? appendDim1 + m2rlen : -1;
+ appendDim2 = (m2clen>0) ? m2clen : appendDim2;
+ }
+ }
+ output.setDimensions(appendDim1, appendDim2);
output.setBlockDimensions (id.getRowsInBlock(), id.getColumnsInBlock());
break;
http://git-wip-us.apache.org/repos/asf/systemml/blob/c0b6ef5c/src/main/java/org/apache/sysml/parser/DMLTranslator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/DMLTranslator.java b/src/main/java/org/apache/sysml/parser/DMLTranslator.java
index fb8404f..0d466e2 100644
--- a/src/main/java/org/apache/sysml/parser/DMLTranslator.java
+++ b/src/main/java/org/apache/sysml/parser/DMLTranslator.java
@@ -43,7 +43,7 @@ import org.apache.sysml.hops.Hop.AggOp;
import org.apache.sysml.hops.Hop.DataGenMethod;
import org.apache.sysml.hops.Hop.DataOpTypes;
import org.apache.sysml.hops.Hop.Direction;
-import org.apache.sysml.hops.Hop.MultiInputOp;
+import org.apache.sysml.hops.Hop.OpOpN;
import org.apache.sysml.hops.Hop.OpOp2;
import org.apache.sysml.hops.Hop.OpOp3;
import org.apache.sysml.hops.Hop.ParamBuiltinOp;
@@ -53,7 +53,7 @@ import org.apache.sysml.hops.IndexingOp;
import org.apache.sysml.hops.LeftIndexingOp;
import org.apache.sysml.hops.LiteralOp;
import org.apache.sysml.hops.MemoTable;
-import org.apache.sysml.hops.MultipleOp;
+import org.apache.sysml.hops.NaryOp;
import org.apache.sysml.hops.OptimizerUtils;
import org.apache.sysml.hops.ParameterizedBuiltinOp;
import org.apache.sysml.hops.ReorgOp;
@@ -1185,10 +1185,10 @@ public class DMLTranslator
HashMap<String, Hop> ids = new HashMap<>();
ArrayList<Hop> output = new ArrayList<>();
- VariableSet liveIn = sb.liveIn();
+ VariableSet liveIn = sb.liveIn();
VariableSet liveOut = sb.liveOut();
- VariableSet updated = sb._updated;
- VariableSet gen = sb._gen;
+ VariableSet updated = sb._updated;
+ VariableSet gen = sb._gen;
VariableSet updatedLiveOut = new VariableSet();
// handle liveout variables that are updated --> target identifiers for Assignment
@@ -1317,8 +1317,8 @@ public class DMLTranslator
inHops[j] = inHop;
}
target.setValueType(ValueType.STRING);
- Hop printfHop = new MultipleOp(target.getName(), target.getDataType(), target.getValueType(),
- MultiInputOp.PRINTF, inHops);
+ Hop printfHop = new NaryOp(target.getName(), target.getDataType(), target.getValueType(),
+ OpOpN.PRINTF, inHops);
output.add(printfHop);
}
@@ -2663,13 +2663,13 @@ public class DMLTranslator
break;
case CBIND:
- currBuiltinOp = new BinaryOp(target.getName(), target.getDataType(), target.getValueType(),
- Hop.OpOp2.CBIND, expr, expr2);
- break;
-
case RBIND:
- currBuiltinOp = new BinaryOp(target.getName(), target.getDataType(), target.getValueType(),
- Hop.OpOp2.RBIND, expr, expr2);
+ OpOp2 appendOp1 = (source.getOpCode()==BuiltinFunctionOp.CBIND) ? OpOp2.CBIND : OpOp2.RBIND;
+ OpOpN appendOp2 = (source.getOpCode()==BuiltinFunctionOp.CBIND) ? OpOpN.CBIND : OpOpN.RBIND;
+ currBuiltinOp = (source.getAllExpr().length == 2) ?
+ new BinaryOp(target.getName(), target.getDataType(), target.getValueType(), appendOp1, expr, expr2) :
+ new NaryOp(target.getName(), target.getDataType(), target.getValueType(), appendOp2,
+ processAllExpressions(source.getAllExpr(), hops));
break;
case DIAG:
@@ -3098,6 +3098,13 @@ public class DMLTranslator
return currBuiltinOp;
}
+ private Hop[] processAllExpressions(Expression[] expr, HashMap<String, Hop> hops) throws ParseException {
+ Hop[] ret = new Hop[expr.length];
+ for(int i=0; i<expr.length; i++)
+ ret[i] = processExpression(expr[i], null, hops);
+ return ret;
+ }
+
private static void setBlockSizeAndRefreshSizeInfo(Hop in, Hop out) {
out.setOutputBlocksizes(in.getRowsInBlock(), in.getColsInBlock());
out.refreshSizeInformation();
http://git-wip-us.apache.org/repos/asf/systemml/blob/c0b6ef5c/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java b/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java
index ecb9629..3b9c075 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java
@@ -267,7 +267,7 @@ public class ExecutionContext {
* @return matrix block
* @throws DMLRuntimeException if DMLRuntimeException occurs
*/
- private MatrixBlock getMatrixInput(String varName)
+ public MatrixBlock getMatrixInput(String varName)
throws DMLRuntimeException
{
MatrixObject mo = getMatrixObject(varName);
@@ -419,6 +419,11 @@ public class ExecutionContext {
}
}
+ public void releaseMatrixInput(String varName) throws DMLRuntimeException {
+ MatrixObject mo = getMatrixObject(varName);
+ mo.release(null);
+ }
+
public void releaseMatrixInputForGPUInstruction(String varName)
throws DMLRuntimeException
{
@@ -485,10 +490,11 @@ public class ExecutionContext {
mo.getGPUObject(getGPUContext(0)).releaseOutput();
}
+ public void setMatrixOutput(String varName, MatrixBlock outputData) throws DMLRuntimeException {
+ setMatrixOutput(varName, outputData, null);
+ }
- public void setMatrixOutput(String varName, MatrixBlock outputData, String opcode)
- throws DMLRuntimeException
- {
+ public void setMatrixOutput(String varName, MatrixBlock outputData, String opcode) throws DMLRuntimeException {
MatrixObject mo = getMatrixObject(varName);
mo.acquireModify(outputData, opcode);
mo.release(opcode);
http://git-wip-us.apache.org/repos/asf/systemml/blob/c0b6ef5c/src/main/java/org/apache/sysml/runtime/instructions/CPInstructionParser.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/CPInstructionParser.java b/src/main/java/org/apache/sysml/runtime/instructions/CPInstructionParser.java
index d0bc429..97e1bcd 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/CPInstructionParser.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/CPInstructionParser.java
@@ -37,7 +37,7 @@ import org.apache.sysml.runtime.instructions.cp.ArithmeticBinaryCPInstruction;
import org.apache.sysml.runtime.instructions.cp.BooleanBinaryCPInstruction;
import org.apache.sysml.runtime.instructions.cp.BooleanUnaryCPInstruction;
import org.apache.sysml.runtime.instructions.cp.BuiltinBinaryCPInstruction;
-import org.apache.sysml.runtime.instructions.cp.BuiltinMultipleCPInstruction;
+import org.apache.sysml.runtime.instructions.cp.BuiltinNary;
import org.apache.sysml.runtime.instructions.cp.BuiltinUnaryCPInstruction;
import org.apache.sysml.runtime.instructions.cp.CPInstruction;
import org.apache.sysml.runtime.instructions.cp.CPInstruction.CPINSTRUCTION_TYPE;
@@ -181,7 +181,9 @@ public class CPInstructionParser extends InstructionParser
String2CPInstructionType.put( "sigmoid", CPINSTRUCTION_TYPE.BuiltinUnary);
String2CPInstructionType.put( "sel+", CPINSTRUCTION_TYPE.BuiltinUnary);
- String2CPInstructionType.put( "printf" , CPINSTRUCTION_TYPE.BuiltinMultiple);
+ String2CPInstructionType.put( "printf" , CPINSTRUCTION_TYPE.BuiltinNary);
+ String2CPInstructionType.put( "cbind" , CPINSTRUCTION_TYPE.BuiltinNary);
+ String2CPInstructionType.put( "rbind" , CPINSTRUCTION_TYPE.BuiltinNary);
// Parameterized Builtin Functions
String2CPInstructionType.put( "cdf" , CPINSTRUCTION_TYPE.ParameterizedBuiltin);
@@ -343,8 +345,10 @@ public class CPInstructionParser extends InstructionParser
case BuiltinUnary:
return BuiltinUnaryCPInstruction.parseInstruction(str);
- case BuiltinMultiple:
- return BuiltinMultipleCPInstruction.parseInstruction(str);
+
+ case BuiltinNary:
+ return BuiltinNary.parseInstruction(str);
+
case Reorg:
return ReorgCPInstruction.parseInstruction(str);
@@ -355,7 +359,7 @@ public class CPInstructionParser extends InstructionParser
return UaggOuterChainCPInstruction.parseInstruction(str);
case MatrixReshape:
- return MatrixReshapeCPInstruction.parseInstruction(str);
+ return MatrixReshapeCPInstruction.parseInstruction(str);
case Append:
return AppendCPInstruction.parseInstruction(str);
http://git-wip-us.apache.org/repos/asf/systemml/blob/c0b6ef5c/src/main/java/org/apache/sysml/runtime/instructions/SPInstructionParser.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/SPInstructionParser.java b/src/main/java/org/apache/sysml/runtime/instructions/SPInstructionParser.java
index bff75d7..1a5033e 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/SPInstructionParser.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/SPInstructionParser.java
@@ -46,6 +46,7 @@ import org.apache.sysml.runtime.instructions.spark.AppendRSPInstruction;
import org.apache.sysml.runtime.instructions.spark.ArithmeticBinarySPInstruction;
import org.apache.sysml.runtime.instructions.spark.BinUaggChainSPInstruction;
import org.apache.sysml.runtime.instructions.spark.BuiltinBinarySPInstruction;
+import org.apache.sysml.runtime.instructions.spark.BuiltinNarySPInstruction;
import org.apache.sysml.runtime.instructions.spark.BuiltinUnarySPInstruction;
import org.apache.sysml.runtime.instructions.spark.CSVReblockSPInstruction;
import org.apache.sysml.runtime.instructions.spark.CastSPInstruction;
@@ -243,6 +244,8 @@ public class SPInstructionParser extends InstructionParser
String2SPInstructionType.put( "rappend", SPINSTRUCTION_TYPE.RAppend);
String2SPInstructionType.put( "gappend", SPINSTRUCTION_TYPE.GAppend);
String2SPInstructionType.put( "galignedappend", SPINSTRUCTION_TYPE.GAlignedAppend);
+ String2SPInstructionType.put( "cbind", SPINSTRUCTION_TYPE.BuiltinNary);
+ String2SPInstructionType.put( "rbind", SPINSTRUCTION_TYPE.BuiltinNary);
String2SPInstructionType.put( DataGen.RAND_OPCODE , SPINSTRUCTION_TYPE.Rand);
String2SPInstructionType.put( DataGen.SEQ_OPCODE , SPINSTRUCTION_TYPE.Rand);
@@ -399,7 +402,10 @@ public class SPInstructionParser extends InstructionParser
case BuiltinUnary:
return BuiltinUnarySPInstruction.parseInstruction(str);
-
+
+ case BuiltinNary:
+ return BuiltinNarySPInstruction.parseInstruction(str);
+
case ParameterizedBuiltin:
return ParameterizedBuiltinSPInstruction.parseInstruction(str);
http://git-wip-us.apache.org/repos/asf/systemml/blob/c0b6ef5c/src/main/java/org/apache/sysml/runtime/instructions/cp/BuiltinMultipleCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/BuiltinMultipleCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/BuiltinMultipleCPInstruction.java
deleted file mode 100644
index 6da8e48..0000000
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/BuiltinMultipleCPInstruction.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.sysml.runtime.instructions.cp;
-
-import java.util.Arrays;
-
-import org.apache.sysml.lops.MultipleCP;
-import org.apache.sysml.runtime.DMLRuntimeException;
-import org.apache.sysml.runtime.functionobjects.Builtin;
-import org.apache.sysml.runtime.functionobjects.ValueFunction;
-import org.apache.sysml.runtime.instructions.InstructionUtils;
-import org.apache.sysml.runtime.matrix.operators.Operator;
-import org.apache.sysml.runtime.matrix.operators.SimpleOperator;
-
-/**
- * Instruction to handle a variable number of input operands. It parses an
- * instruction string to generate an object that is a subclass of
- * BuiltinMultipleCPInstruction. Currently the only subclass of
- * BuiltinMultipleCPInstruction is ScalarBuiltinMultipleCPInstruction. The
- * ScalarBuiltinMultipleCPInstruction class is responsible for printf-style
- * Java-based string formatting.
- *
- */
-public abstract class BuiltinMultipleCPInstruction extends CPInstruction {
-
- public CPOperand output;
- public CPOperand[] inputs;
-
- BuiltinMultipleCPInstruction(Operator op, String opcode, String istr, CPOperand output, CPOperand... inputs) {
- super(op, opcode, istr);
- _cptype = CPINSTRUCTION_TYPE.BuiltinMultiple;
- this.output = output;
- this.inputs = inputs;
- }
-
- public static BuiltinMultipleCPInstruction parseInstruction(String str) throws DMLRuntimeException {
- String[] parts = InstructionUtils.getInstructionPartsWithValueType(str);
-
- String opcode = parts[0];
-
- String outputString = parts[parts.length - 1];
- CPOperand outputOperand = new CPOperand(outputString);
-
- String[] inputStrings = null;
- CPOperand[] inputOperands = null;
- if (parts.length > 2) {
- inputStrings = Arrays.copyOfRange(parts, 1, parts.length - 1);
- inputOperands = new CPOperand[parts.length - 2];
- for (int i = 0; i < inputStrings.length; i++) {
- inputOperands[i] = new CPOperand(inputStrings[i]);
- }
- }
-
- if (MultipleCP.OperationType.PRINTF.toString().equalsIgnoreCase(opcode)) {
- ValueFunction func = Builtin.getBuiltinFnObject(opcode);
- return new ScalarBuiltinMultipleCPInstruction(new SimpleOperator(func), opcode, str, outputOperand,
- inputOperands);
- }
- throw new DMLRuntimeException("Opcode (" + opcode + ") not recognized in BuiltinMultipleCPInstruction");
- }
-}
http://git-wip-us.apache.org/repos/asf/systemml/blob/c0b6ef5c/src/main/java/org/apache/sysml/runtime/instructions/cp/BuiltinNary.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/BuiltinNary.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/BuiltinNary.java
new file mode 100644
index 0000000..1bacc70
--- /dev/null
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/BuiltinNary.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.runtime.instructions.cp;
+
+import org.apache.sysml.lops.Nary;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.functionobjects.Builtin;
+import org.apache.sysml.runtime.functionobjects.ValueFunction;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.matrix.operators.Operator;
+import org.apache.sysml.runtime.matrix.operators.SimpleOperator;
+
+/**
+ * Instruction to handle a variable number of input operands. It parses an
+ * instruction string to generate an object that is a subclass of
+ * BuiltinMultipleCPInstruction. Currently the only subclass of
+ * BuiltinMultipleCPInstruction is ScalarBuiltinMultipleCPInstruction. The
+ * ScalarBuiltinMultipleCPInstruction class is responsible for printf-style
+ * Java-based string formatting.
+ *
+ */
+public abstract class BuiltinNary extends CPInstruction
+{
+ public CPOperand output;
+ public CPOperand[] inputs;
+
+ public BuiltinNary(Operator op, String opcode, String istr, CPOperand output, CPOperand... inputs) {
+ super(op, opcode, istr);
+ _cptype = CPINSTRUCTION_TYPE.BuiltinNary;
+ this.output = output;
+ this.inputs = inputs;
+ }
+
+ public static BuiltinNary parseInstruction(String str) throws DMLRuntimeException {
+ String[] parts = InstructionUtils.getInstructionPartsWithValueType(str);
+ String opcode = parts[0];
+ CPOperand outputOperand = new CPOperand(parts[parts.length - 1]);
+ CPOperand[] inputOperands = null;
+ if (parts.length > 2) {
+ inputOperands = new CPOperand[parts.length - 2];
+ for (int i = 1; i < parts.length-1; i++)
+ inputOperands[i-1] = new CPOperand(parts[i]);
+ }
+
+ if( Nary.OperationType.PRINTF.name().equalsIgnoreCase(opcode) ) {
+ ValueFunction func = Builtin.getBuiltinFnObject(opcode);
+ return new ScalarBuiltinNaryCPInstruction(new SimpleOperator(func),
+ opcode, str, outputOperand, inputOperands);
+ }
+ else if( opcode.equals("cbind") || opcode.equals("rbind") ) {
+ return new MatrixBuiltinNaryCPInstruction(null,
+ opcode, str, outputOperand, inputOperands);
+ }
+
+ throw new DMLRuntimeException("Opcode (" + opcode + ") not recognized in BuiltinMultipleCPInstruction");
+ }
+}
http://git-wip-us.apache.org/repos/asf/systemml/blob/c0b6ef5c/src/main/java/org/apache/sysml/runtime/instructions/cp/CPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/CPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/CPInstruction.java
index 46251e9..d7fc623 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/CPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/CPInstruction.java
@@ -31,8 +31,8 @@ public abstract class CPInstruction extends Instruction
{
public enum CPINSTRUCTION_TYPE { INVALID,
AggregateUnary, AggregateBinary, AggregateTernary, ArithmeticBinary,
- Ternary, Quaternary, BooleanBinary, BooleanUnary, BuiltinBinary, BuiltinUnary,
- BuiltinMultiple, MultiReturnParameterizedBuiltin, ParameterizedBuiltin, MultiReturnBuiltin,
+ Ternary, Quaternary, BooleanBinary, BooleanUnary, BuiltinBinary, BuiltinUnary, BuiltinNary,
+ MultiReturnParameterizedBuiltin, ParameterizedBuiltin, MultiReturnBuiltin,
Builtin, Reorg, RelationalBinary, Variable, External, Append, Rand, QSort, QPick,
MatrixIndexing, MMTSJ, PMMJ, MMChain, MatrixReshape, Partition, Compression, SpoofFused,
StringInit, CentralMoment, Covariance, UaggOuterChain, Convolution }
http://git-wip-us.apache.org/repos/asf/systemml/blob/c0b6ef5c/src/main/java/org/apache/sysml/runtime/instructions/cp/MatrixBuiltinNaryCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/MatrixBuiltinNaryCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/MatrixBuiltinNaryCPInstruction.java
new file mode 100644
index 0000000..8ad619b
--- /dev/null
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/MatrixBuiltinNaryCPInstruction.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.runtime.instructions.cp;
+
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.operators.Operator;
+
+public class MatrixBuiltinNaryCPInstruction extends BuiltinNary {
+
+ protected MatrixBuiltinNaryCPInstruction(Operator op, String opcode, String istr, CPOperand output, CPOperand[] inputs) {
+ super(op, opcode, istr, output, inputs);
+ }
+
+ @Override
+ public void processInstruction(ExecutionContext ec) throws DMLRuntimeException {
+ //pin input matrix blocks
+ MatrixBlock in1 = ec.getMatrixInput(inputs[0].getName());
+ MatrixBlock[] in2 = new MatrixBlock[inputs.length-1];
+ for( int i=1; i<inputs.length; i++ )
+ in2[i-1] = ec.getMatrixInput(inputs[i].getName());
+
+ MatrixBlock outBlock = null;
+ if( "cbind".equals(getOpcode()) || "rbind".equals(getOpcode()) ) {
+ boolean cbind = "cbind".equals(getOpcode());
+ outBlock = in1.appendOperations(in2, new MatrixBlock(), cbind);
+ }
+ else {
+ throw new DMLRuntimeException("Unknown opcode: "+getOpcode());
+ }
+
+ //release inputs and set output
+ for( int i=0; i<inputs.length; i++ )
+ ec.releaseMatrixInput(inputs[i].getName());
+ ec.setMatrixOutput(output.getName(), outBlock);
+ }
+}
http://git-wip-us.apache.org/repos/asf/systemml/blob/c0b6ef5c/src/main/java/org/apache/sysml/runtime/instructions/cp/ScalarBuiltinMultipleCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/ScalarBuiltinMultipleCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/ScalarBuiltinMultipleCPInstruction.java
deleted file mode 100644
index 8169695..0000000
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/ScalarBuiltinMultipleCPInstruction.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.sysml.runtime.instructions.cp;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.sysml.api.DMLScript;
-import org.apache.sysml.lops.MultipleCP;
-import org.apache.sysml.parser.Expression;
-import org.apache.sysml.runtime.DMLRuntimeException;
-import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
-import org.apache.sysml.runtime.matrix.operators.Operator;
-
-/**
- * The ScalarBuiltinMultipleCPInstruction class is responsible for printf-style
- * Java-based string formatting. The first input is the format string. The
- * inputs after the first input are the arguments to be formatted in the format
- * string.
- *
- */
-public class ScalarBuiltinMultipleCPInstruction extends BuiltinMultipleCPInstruction {
-
- protected ScalarBuiltinMultipleCPInstruction(Operator op, String opcode, String istr, CPOperand output,
- CPOperand... inputs) {
- super(op, opcode, istr, output, inputs);
- }
-
- @Override
- public void processInstruction(ExecutionContext ec) throws DMLRuntimeException {
- if (MultipleCP.OperationType.PRINTF.toString().equalsIgnoreCase(getOpcode())) {
- List<ScalarObject> scalarObjects = new ArrayList<>();
- for (CPOperand input : inputs) {
- ScalarObject so = ec.getScalarInput(input.getName(), input.getValueType(), input.isLiteral());
- scalarObjects.add(so);
- }
-
- // determine the format string (first argument) to pass to String.format
- ScalarObject formatStringObject = scalarObjects.get(0);
- if (formatStringObject.getValueType() != Expression.ValueType.STRING) {
- throw new DMLRuntimeException("First parameter needs to be a string");
- }
- String formatString = formatStringObject.getStringValue();
-
- // determine the arguments after the format string to pass to String.format
- Object[] objects = null;
- if (scalarObjects.size() > 1) {
- objects = new Object[scalarObjects.size() - 1];
- for (int i = 1; i < scalarObjects.size(); i++) {
- ScalarObject scalarObject = scalarObjects.get(i);
- switch (scalarObject.getValueType()) {
- case INT:
- objects[i - 1] = scalarObject.getLongValue();
- break;
- case DOUBLE:
- objects[i - 1] = scalarObject.getDoubleValue();
- break;
- case BOOLEAN:
- objects[i - 1] = scalarObject.getBooleanValue();
- break;
- case STRING:
- objects[i - 1] = scalarObject.getStringValue();
- break;
- default:
- }
- }
- }
-
- String result = String.format(formatString, objects);
- if (!DMLScript.suppressPrint2Stdout()) {
- System.out.println(result);
- }
-
- // this is necessary so that the remove variable operation can be
- // performed
- ec.setScalarOutput(output.getName(), new StringObject(result));
- } else {
- throw new DMLRuntimeException(
- "Opcode (" + getOpcode() + ") not recognized in ScalarBuiltinMultipleCPInstruction");
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/systemml/blob/c0b6ef5c/src/main/java/org/apache/sysml/runtime/instructions/cp/ScalarBuiltinNaryCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/ScalarBuiltinNaryCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/ScalarBuiltinNaryCPInstruction.java
new file mode 100644
index 0000000..9b15696
--- /dev/null
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/ScalarBuiltinNaryCPInstruction.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.runtime.instructions.cp;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.lops.Nary;
+import org.apache.sysml.parser.Expression;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.matrix.operators.Operator;
+
+/**
+ * The ScalarBuiltinMultipleCPInstruction class is responsible for printf-style
+ * Java-based string formatting. The first input is the format string. The
+ * inputs after the first input are the arguments to be formatted in the format
+ * string.
+ *
+ */
+public class ScalarBuiltinNaryCPInstruction extends BuiltinNary {
+
+ protected ScalarBuiltinNaryCPInstruction(Operator op, String opcode, String istr, CPOperand output, CPOperand[] inputs) {
+ super(op, opcode, istr, output, inputs);
+ }
+
+ @Override
+ public void processInstruction(ExecutionContext ec) throws DMLRuntimeException {
+ if (Nary.OperationType.PRINTF.toString().equalsIgnoreCase(getOpcode())) {
+ List<ScalarObject> scalarObjects = new ArrayList<>();
+ for (CPOperand input : inputs) {
+ ScalarObject so = ec.getScalarInput(input.getName(), input.getValueType(), input.isLiteral());
+ scalarObjects.add(so);
+ }
+
+ // determine the format string (first argument) to pass to String.format
+ ScalarObject formatStringObject = scalarObjects.get(0);
+ if (formatStringObject.getValueType() != Expression.ValueType.STRING) {
+ throw new DMLRuntimeException("First parameter needs to be a string");
+ }
+ String formatString = formatStringObject.getStringValue();
+
+ // determine the arguments after the format string to pass to String.format
+ Object[] objects = null;
+ if (scalarObjects.size() > 1) {
+ objects = new Object[scalarObjects.size() - 1];
+ for (int i = 1; i < scalarObjects.size(); i++) {
+ ScalarObject scalarObject = scalarObjects.get(i);
+ switch (scalarObject.getValueType()) {
+ case INT:
+ objects[i - 1] = scalarObject.getLongValue();
+ break;
+ case DOUBLE:
+ objects[i - 1] = scalarObject.getDoubleValue();
+ break;
+ case BOOLEAN:
+ objects[i - 1] = scalarObject.getBooleanValue();
+ break;
+ case STRING:
+ objects[i - 1] = scalarObject.getStringValue();
+ break;
+ default:
+ }
+ }
+ }
+
+ String result = String.format(formatString, objects);
+ if (!DMLScript.suppressPrint2Stdout()) {
+ System.out.println(result);
+ }
+
+ // this is necessary so that the remove variable operation can be
+ // performed
+ ec.setScalarOutput(output.getName(), new StringObject(result));
+ } else {
+ throw new DMLRuntimeException(
+ "Opcode (" + getOpcode() + ") not recognized in ScalarBuiltinMultipleCPInstruction");
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/systemml/blob/c0b6ef5c/src/main/java/org/apache/sysml/runtime/instructions/spark/BuiltinNarySPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/BuiltinNarySPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/BuiltinNarySPInstruction.java
new file mode 100644
index 0000000..4edb1cf
--- /dev/null
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/BuiltinNarySPInstruction.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.runtime.instructions.spark;
+
+import org.apache.spark.api.java.JavaPairRDD;
+import org.apache.spark.api.java.function.PairFunction;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.instructions.cp.CPOperand;
+import org.apache.sysml.runtime.instructions.spark.AppendGSPInstruction.ShiftMatrix;
+import org.apache.sysml.runtime.instructions.spark.utils.RDDAggregateUtils;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.util.UtilFunctions;
+
+import scala.Tuple2;
+
+public class BuiltinNarySPInstruction extends SPInstruction
+{
+ private CPOperand[] inputs;
+ private CPOperand output;
+
+ protected BuiltinNarySPInstruction(CPOperand[] in, CPOperand out, String opcode, String istr) {
+ super(opcode, istr);
+ _sptype = SPINSTRUCTION_TYPE.BuiltinNary;
+ inputs = in;
+ output = out;
+ }
+
+ public static BuiltinNarySPInstruction parseInstruction ( String str )
+ throws DMLRuntimeException
+ {
+ String[] parts = InstructionUtils.getInstructionPartsWithValueType(str);
+ String opcode = parts[0];
+ CPOperand output = new CPOperand(parts[parts.length - 1]);
+ CPOperand[] inputs = null;
+ inputs = new CPOperand[parts.length - 2];
+ for (int i = 1; i < parts.length-1; i++)
+ inputs[i-1] = new CPOperand(parts[i]);
+ return new BuiltinNarySPInstruction(inputs, output, opcode, str);
+ }
+
+ @Override
+ public void processInstruction(ExecutionContext ec)
+ throws DMLRuntimeException
+ {
+ SparkExecutionContext sec = (SparkExecutionContext)ec;
+ boolean cbind = getOpcode().equals("cbind");
+
+ //compute output characteristics
+ MatrixCharacteristics mcOut = computeOutputMatrixCharacteristics(sec, inputs, cbind);
+
+ //get consolidated input via union over shifted and padded inputs
+ MatrixCharacteristics off = new MatrixCharacteristics(
+ 0, 0, mcOut.getRowsPerBlock(), mcOut.getColsPerBlock(), 0);
+ JavaPairRDD<MatrixIndexes,MatrixBlock> out = null;
+ for( CPOperand input : inputs ) {
+ MatrixCharacteristics mcIn = sec.getMatrixCharacteristics(input.getName());
+ JavaPairRDD<MatrixIndexes,MatrixBlock> in = sec
+ .getBinaryBlockRDDHandleForVariable( input.getName() )
+ .flatMapToPair(new ShiftMatrix(off, mcIn, cbind))
+ .mapToPair(new PadBlocksFunction(mcOut)); //just padding
+ out = (out != null) ? out.union(in) : in;
+ updateMatrixCharacteristics(mcIn, off, cbind);
+ }
+
+ //aggregate partially overlapping blocks w/ single shuffle
+ out = RDDAggregateUtils.mergeByKey(out);
+
+ //set output RDD and add lineage
+ sec.getMatrixCharacteristics(output.getName()).set(mcOut);
+ sec.setRDDHandleForVariable(output.getName(), out);
+ for( CPOperand input : inputs )
+ sec.addLineageRDD(output.getName(), input.getName());
+ }
+
+ private static MatrixCharacteristics computeOutputMatrixCharacteristics(SparkExecutionContext sec, CPOperand[] inputs, boolean cbind)
+ throws DMLRuntimeException
+ {
+ MatrixCharacteristics mcIn1 = sec.getMatrixCharacteristics(inputs[0].getName());
+ MatrixCharacteristics mcOut = new MatrixCharacteristics(
+ 0, 0, mcIn1.getRowsPerBlock(), mcIn1.getColsPerBlock(), 0);
+ for( CPOperand input : inputs ) {
+ MatrixCharacteristics mcIn = sec.getMatrixCharacteristics(input.getName());
+ updateMatrixCharacteristics(mcIn, mcOut, cbind);
+ }
+ return mcOut;
+ }
+
+ private static void updateMatrixCharacteristics(MatrixCharacteristics in, MatrixCharacteristics out, boolean cbind) {
+ out.setDimension(cbind ? Math.max(out.getRows(), in.getRows()) : out.getRows()+in.getRows(),
+ cbind ? out.getCols()+in.getCols() : Math.max(out.getCols(), in.getCols()));
+ out.setNonZeros((out.getNonZeros()!=-1 && in.dimsKnown(true)) ? out.getNonZeros()+in.getNonZeros() : -1);
+ }
+
+ public static class PadBlocksFunction implements PairFunction<Tuple2<MatrixIndexes,MatrixBlock>,MatrixIndexes,MatrixBlock>
+ {
+ private static final long serialVersionUID = 1291358959908299855L;
+
+ private final MatrixCharacteristics _mcOut;
+
+ public PadBlocksFunction(MatrixCharacteristics mcOut) {
+ _mcOut = mcOut;
+ }
+
+ public Tuple2<MatrixIndexes, MatrixBlock> call(Tuple2<MatrixIndexes, MatrixBlock> arg0) throws Exception {
+ MatrixIndexes ix = arg0._1();
+ MatrixBlock mb = arg0._2();
+ int brlen = UtilFunctions.computeBlockSize(_mcOut.getRows(), ix.getRowIndex(), _mcOut.getRowsPerBlock());
+ int bclen = UtilFunctions.computeBlockSize(_mcOut.getCols(), ix.getColumnIndex(), _mcOut.getColsPerBlock());
+
+ //check for pass-through
+ if( brlen == mb.getNumRows() && bclen == mb.getNumColumns() )
+ return arg0;
+
+ //cbind or rbind to pad to right blocksize
+ if( brlen > mb.getNumRows() ) //rbind
+ mb = mb.appendOperations(new MatrixBlock(brlen-mb.getNumRows(),bclen,true), new MatrixBlock(), false);
+ else if( bclen > mb.getNumColumns() ) //cbind
+ mb = mb.appendOperations(new MatrixBlock(brlen,bclen-mb.getNumColumns(),true), new MatrixBlock(), true);
+ return new Tuple2<>(ix, mb);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/systemml/blob/c0b6ef5c/src/main/java/org/apache/sysml/runtime/instructions/spark/SPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/SPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/SPInstruction.java
index f0ee6e3..bafe4e8 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/SPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/SPInstruction.java
@@ -32,7 +32,7 @@ public abstract class SPInstruction extends Instruction {
public enum SPINSTRUCTION_TYPE {
MAPMM, MAPMMCHAIN, CPMM, RMM, TSMM, TSMM2, PMM, ZIPMM, PMAPMM, //matrix multiplication instructions
MatrixIndexing, Reorg, ArithmeticBinary, RelationalBinary, AggregateUnary, AggregateTernary, Reblock, CSVReblock,
- Builtin, BuiltinUnary, BuiltinBinary, MultiReturnBuiltin, Checkpoint, Compression, Cast,
+ Builtin, BuiltinUnary, BuiltinBinary, BuiltinNary, MultiReturnBuiltin, Checkpoint, Compression, Cast,
CentralMoment, Covariance, QSort, QPick,
ParameterizedBuiltin, MAppend, RAppend, GAppend, GAlignedAppend, Rand,
MatrixReshape, Ternary, Quaternary, CumsumAggregate, CumsumOffset, BinUaggChain, UaggOuterChain,
http://git-wip-us.apache.org/repos/asf/systemml/blob/c0b6ef5c/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java
index c023890..48051de 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java
@@ -1650,7 +1650,7 @@ public class MatrixBlock extends MatrixValue implements CacheBlock, Externalizab
nonZeros -= recomputeNonZeros(rl, ru, cl, cu);
copyEmptyToDense(rl, ru, cl, cu);
}
- return;
+ return;
}
//allocate output block
@@ -1661,7 +1661,7 @@ public class MatrixBlock extends MatrixValue implements CacheBlock, Externalizab
nonZeros = nonZeros - recomputeNonZeros(rl, ru, cl, cu) + src.nonZeros;
//copy values
- int rowLen = cu-cl+1;
+ int rowLen = cu-cl+1;
if(clen == src.clen) //optimization for equal width
System.arraycopy(src.denseBlock, 0, denseBlock, rl*clen+cl, src.rlen*src.clen);
else
@@ -3521,24 +3521,25 @@ public class MatrixBlock extends MatrixValue implements CacheBlock, Externalizab
return result;
}
- public MatrixBlock appendOperations( MatrixBlock that, MatrixBlock ret )
- throws DMLRuntimeException
- {
- //default append-cbind
- return appendOperations(that, ret, true);
+ public MatrixBlock appendOperations( MatrixBlock that, MatrixBlock ret ) throws DMLRuntimeException {
+ return appendOperations(that, ret, true); //default cbind
}
- public MatrixBlock appendOperations( MatrixBlock that, MatrixBlock ret, boolean cbind )
+ public MatrixBlock appendOperations( MatrixBlock that, MatrixBlock ret, boolean cbind ) throws DMLRuntimeException {
+ return appendOperations(new MatrixBlock[]{that}, ret, cbind);
+ }
+
+ public MatrixBlock appendOperations( MatrixBlock[] that, MatrixBlock ret, boolean cbind )
throws DMLRuntimeException
{
MatrixBlock result = checkType( ret );
- final int m = cbind ? rlen : rlen+that.rlen;
- final int n = cbind ? clen+that.clen : clen;
- final long nnz = nonZeros+that.nonZeros;
+ final int m = cbind ? rlen : rlen+Arrays.stream(that).mapToInt(mb -> mb.rlen).sum();
+ final int n = cbind ? clen+Arrays.stream(that).mapToInt(mb -> mb.clen).sum() : clen;
+ final long nnz = nonZeros+Arrays.stream(that).mapToLong(mb -> mb.nonZeros).sum();
boolean sp = evalSparseFormatInMemory(m, n, nnz);
//init result matrix
- if( result == null )
+ if( result == null )
result = new MatrixBlock(m, n, sp, nnz);
else
result.reset(m, n, sp, nnz);
@@ -3546,54 +3547,67 @@ public class MatrixBlock extends MatrixValue implements CacheBlock, Externalizab
//core append operation
//copy left and right input into output
if( !result.sparse ) //DENSE
- {
+ {
if( cbind ) {
result.copy(0, m-1, 0, clen-1, this, false);
- result.copy(0, m-1, clen, n-1, that, false);
+ for(int i=0, off=clen; i<that.length; i++) {
+ result.copy(0, m-1, off, off+that[i].clen-1, that[i], false);
+ off += that[i].clen;
+ }
}
else { //rbind
result.copy(0, rlen-1, 0, n-1, this, false);
- result.copy(rlen, m-1, 0, n-1, that, false);
+ for(int i=0, off=rlen; i<that.length; i++) {
+ result.copy(off, off+that[i].rlen-1, 0, n-1, that[i], false);
+ off += that[i].rlen;
+ }
}
}
else //SPARSE
{
//adjust sparse rows if required
- if( !this.isEmptyBlock(false) || !that.isEmptyBlock(false) ) {
+ if( !this.isEmptyBlock(false) || !Arrays.stream(that).allMatch(mb -> mb.isEmptyBlock(false)) ) {
result.allocateSparseRowsBlock();
-
//allocate sparse rows once for cbind
if( cbind && result.getSparseBlock() instanceof SparseBlockMCSR ) {
SparseBlock sblock = result.getSparseBlock();
for( int i=0; i<result.rlen; i++ ) {
- int lnnz = (int)(this.recomputeNonZeros(i, i, 0, this.clen-1)
- + that.recomputeNonZeros(i, i, 0, that.clen-1));
+ final int row = i; //workaround for lambda compile issue
+ int lnnz = (int) (this.recomputeNonZeros(i, i, 0, this.clen-1) + Arrays.stream(that)
+ .mapToLong(mb -> mb.recomputeNonZeros(row, row, 0, mb.clen-1)).sum());
sblock.allocate(i, lnnz);
}
}
}
//core append operation
- result.appendToSparse(this, 0, 0);
- if( cbind )
- result.appendToSparse(that, 0, clen);
- else //rbind
- result.appendToSparse(that, rlen, 0);
- }
+ result.appendToSparse(this, 0, 0);
+ if( cbind ) {
+ for(int i=0, off=clen; i<that.length; i++) {
+ result.appendToSparse(that[i], 0, off);
+ off += that[i].clen;
+ }
+ }
+ else { //rbind
+ for(int i=0, off=rlen; i<that.length; i++) {
+ result.appendToSparse(that[i], off, 0);
+ off += that[i].rlen;
+ }
+ }
+ }
//update meta data
result.nonZeros = nnz;
-
return result;
}
- public MatrixBlock transposeSelfMatrixMultOperations( MatrixBlock out, MMTSJType tstype )
+ public MatrixBlock transposeSelfMatrixMultOperations( MatrixBlock out, MMTSJType tstype )
throws DMLRuntimeException
{
return transposeSelfMatrixMultOperations(out, tstype, 1);
}
- public MatrixBlock transposeSelfMatrixMultOperations( MatrixBlock out, MMTSJType tstype, int k )
+ public MatrixBlock transposeSelfMatrixMultOperations( MatrixBlock out, MMTSJType tstype, int k )
throws DMLRuntimeException
{
//check for transpose type