You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by sh...@apache.org on 2015/11/25 23:35:44 UTC
[1/5] incubator-systemml git commit: Enabling transform functionality
in standalone mode
Repository: incubator-systemml
Updated Branches:
refs/heads/master 52ec04aff -> 5c88842d1
Enabling transform functionality in standalone mode
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/8040be60
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/8040be60
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/8040be60
Branch: refs/heads/master
Commit: 8040be6005b0ac0c3f2cac5dfb6edeed365c0c60
Parents: a2f78e7
Author: shirisht <st...@us.ibm.com>
Authored: Wed Nov 25 00:34:31 2015 -0800
Committer: shirisht <st...@us.ibm.com>
Committed: Wed Nov 25 00:42:22 2015 -0800
----------------------------------------------------------------------
.../java/com/ibm/bi/dml/lops/compile/Dag.java | 13 ++-
.../ParameterizedBuiltinFunctionExpression.java | 2 +-
.../cp/ParameterizedBuiltinCPInstruction.java | 21 +++++
.../instructions/cp/VariableCPInstruction.java | 27 ++++++
.../bi/dml/runtime/transform/DataTransform.java | 34 ++++++-
.../functions/transform/TransformTest.java | 96 ++++++++++++++++++++
6 files changed, 188 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/8040be60/src/main/java/com/ibm/bi/dml/lops/compile/Dag.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/compile/Dag.java b/src/main/java/com/ibm/bi/dml/lops/compile/Dag.java
index 282ead6..e5b63f5 100644
--- a/src/main/java/com/ibm/bi/dml/lops/compile/Dag.java
+++ b/src/main/java/com/ibm/bi/dml/lops/compile/Dag.java
@@ -2875,9 +2875,16 @@ public class Dag<N extends Lop>
oparams.setLabel(Lop.MATRIX_VAR_NAME_PREFIX + var_index.getNextID());
// generate an instruction that creates a symbol table entry for the new variable in CSV format
- Instruction createvarInst = VariableCPInstruction.parseInstruction(
- dataInput.getCreateVarInstructions(
- oparams.getFile_name(), oparams.getLabel()) );
+ Data delimLop = (Data) dataInput.getNamedInputLop(DataExpression.DELIM_DELIMITER);
+
+ Instruction createvarInst = VariableCPInstruction.prepareCreateVariableInstruction(
+ oparams.getLabel(),
+ oparams.getFile_name(),
+ true,
+ OutputInfo.outputInfoToString(OutputInfo.CSVOutputInfo),
+ new MatrixCharacteristics(oparams.getNumRows(), oparams.getNumCols(), -1, -1, oparams.getNnz()),
+ false, delimLop.getStringValue(), true
+ );
createvarInst.setLocation(node);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/8040be60/src/main/java/com/ibm/bi/dml/parser/ParameterizedBuiltinFunctionExpression.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/parser/ParameterizedBuiltinFunctionExpression.java b/src/main/java/com/ibm/bi/dml/parser/ParameterizedBuiltinFunctionExpression.java
index c5dbd1d..979cd22 100644
--- a/src/main/java/com/ibm/bi/dml/parser/ParameterizedBuiltinFunctionExpression.java
+++ b/src/main/java/com/ibm/bi/dml/parser/ParameterizedBuiltinFunctionExpression.java
@@ -242,7 +242,7 @@ public class ParameterizedBuiltinFunctionExpression extends DataIdentifier
raiseValidateError("Named parameter '" + TF_FN_PARAM_DATA + "' missing. Please specify the input data set.", conditional, LanguageErrorCodes.INVALID_PARAMETERS);
}
else if( data.getOutput().getDataType() != DataType.FRAME ){
- raiseValidateError("Input to tansform() must be of type 'table'. It is of type '"+data.getOutput().getDataType()+"'.", conditional, LanguageErrorCodes.INVALID_PARAMETERS);
+ raiseValidateError("Input to tansform() must be of type 'frame'. It is of type '"+data.getOutput().getDataType()+"'.", conditional, LanguageErrorCodes.INVALID_PARAMETERS);
}
Expression txmtd = getVarParam(TF_FN_PARAM_TXMTD);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/8040be60/src/main/java/com/ibm/bi/dml/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java b/src/main/java/com/ibm/bi/dml/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java
index e6b677d..1b14d05 100644
--- a/src/main/java/com/ibm/bi/dml/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java
+++ b/src/main/java/com/ibm/bi/dml/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java
@@ -17,21 +17,27 @@
package com.ibm.bi.dml.runtime.instructions.cp;
+import java.io.IOException;
import java.util.HashMap;
+import org.apache.wink.json4j.JSONException;
+
import com.ibm.bi.dml.lops.Lop;
import com.ibm.bi.dml.parser.Statement;
import com.ibm.bi.dml.runtime.DMLRuntimeException;
import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
+import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
import com.ibm.bi.dml.runtime.functionobjects.ParameterizedBuiltin;
import com.ibm.bi.dml.runtime.functionobjects.ValueFunction;
import com.ibm.bi.dml.runtime.instructions.Instruction;
import com.ibm.bi.dml.runtime.instructions.InstructionUtils;
import com.ibm.bi.dml.runtime.instructions.mr.GroupedAggregateInstruction;
+import com.ibm.bi.dml.runtime.matrix.JobReturn;
import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
import com.ibm.bi.dml.runtime.matrix.operators.Operator;
import com.ibm.bi.dml.runtime.matrix.operators.SimpleOperator;
+import com.ibm.bi.dml.runtime.transform.DataTransform;
public class ParameterizedBuiltinCPInstruction extends ComputationCPInstruction
@@ -217,6 +223,21 @@ public class ParameterizedBuiltinCPInstruction extends ComputationCPInstruction
ec.setMatrixOutput(output.getName(), ret);
ec.releaseMatrixInput(params.get("target"));
}
+ else if ( opcode.equalsIgnoreCase("transform")) {
+ MatrixObject mo = (MatrixObject) ec.getVariable(params.get("target"));
+ MatrixObject out = (MatrixObject) ec.getVariable(output.getName());
+
+ try {
+ JobReturn jt = DataTransform.cpDataTransform(this, new MatrixObject[] { mo } , new MatrixObject[] {out} );
+ out.updateMatrixCharacteristics(jt.getMatrixCharacteristics(0));
+ } catch (IllegalArgumentException e) {
+ throw new DMLRuntimeException(e);
+ } catch (IOException e) {
+ throw new DMLRuntimeException(e);
+ } catch (JSONException e) {
+ throw new DMLRuntimeException(e);
+ }
+ }
else {
throw new DMLRuntimeException("Unknown opcode : " + opcode);
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/8040be60/src/main/java/com/ibm/bi/dml/runtime/instructions/cp/VariableCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/instructions/cp/VariableCPInstruction.java b/src/main/java/com/ibm/bi/dml/runtime/instructions/cp/VariableCPInstruction.java
index 5ecef9a..7d19c9d 100644
--- a/src/main/java/com/ibm/bi/dml/runtime/instructions/cp/VariableCPInstruction.java
+++ b/src/main/java/com/ibm/bi/dml/runtime/instructions/cp/VariableCPInstruction.java
@@ -954,6 +954,33 @@ public class VariableCPInstruction extends CPInstruction
return parseInstruction(str);
}
+ public static Instruction prepareCreateVariableInstruction(String varName, String fileName, boolean fNameOverride, String format, MatrixCharacteristics mc, boolean hasHeader, String delim, boolean sparse) throws DMLRuntimeException, DMLUnsupportedOperationException {
+ StringBuilder sb = new StringBuilder();
+ sb.append(getBasicCreateVarString(varName, fileName, fNameOverride, format));
+
+ sb.append(Lop.OPERAND_DELIMITOR);
+ sb.append(mc.getRows());
+ sb.append(Lop.OPERAND_DELIMITOR);
+ sb.append(mc.getCols());
+ sb.append(Lop.OPERAND_DELIMITOR);
+ sb.append(mc.getRowsPerBlock());
+ sb.append(Lop.OPERAND_DELIMITOR);
+ sb.append(mc.getColsPerBlock());
+ sb.append(Lop.OPERAND_DELIMITOR);
+ sb.append(mc.getNonZeros());
+
+ sb.append(Lop.OPERAND_DELIMITOR);
+ sb.append(hasHeader);
+ sb.append(Lop.OPERAND_DELIMITOR);
+ sb.append(delim);
+ sb.append(Lop.OPERAND_DELIMITOR);
+ sb.append(sparse);
+
+ String str = sb.toString();
+
+ return parseInstruction(str);
+ }
+
@Override
public void updateInstructionThreadID(String pattern, String replace)
throws DMLRuntimeException
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/8040be60/src/main/java/com/ibm/bi/dml/runtime/transform/DataTransform.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/transform/DataTransform.java b/src/main/java/com/ibm/bi/dml/runtime/transform/DataTransform.java
index 33a530e..9479b2f 100644
--- a/src/main/java/com/ibm/bi/dml/runtime/transform/DataTransform.java
+++ b/src/main/java/com/ibm/bi/dml/runtime/transform/DataTransform.java
@@ -59,6 +59,7 @@ import com.ibm.bi.dml.runtime.controlprogram.context.SparkExecutionContext;
import com.ibm.bi.dml.runtime.instructions.Instruction;
import com.ibm.bi.dml.runtime.instructions.InstructionParser;
import com.ibm.bi.dml.runtime.instructions.MRJobInstruction;
+import com.ibm.bi.dml.runtime.instructions.cp.ParameterizedBuiltinCPInstruction;
import com.ibm.bi.dml.runtime.instructions.mr.CSVReblockInstruction;
import com.ibm.bi.dml.runtime.instructions.spark.ParameterizedBuiltinSPInstruction;
import com.ibm.bi.dml.runtime.instructions.spark.data.RDDObject;
@@ -644,6 +645,29 @@ public class DataTransform {
inputCSVProperties = (CSVFileFormatProperties)inputMatrix.getFileFormatProperties();
}
+ TransformOperands(ParameterizedBuiltinCPInstruction inst, MatrixObject inputMatrix) {
+ HashMap<String, String> params = inst.getParameterMap();
+
+ inputPath = inputMatrix.getFileName();
+ txMtdPath = params.get(ParameterizedBuiltinFunctionExpression.TF_FN_PARAM_TXMTD);
+
+ if ( params.get(ParameterizedBuiltinFunctionExpression.TF_FN_PARAM_TXSPEC) != null ) {
+ isApply = false;
+ specFile = params.get(ParameterizedBuiltinFunctionExpression.TF_FN_PARAM_TXSPEC);
+ applyTxPath = null;
+ }
+ else if ( params.get(ParameterizedBuiltinFunctionExpression.TF_FN_PARAM_APPLYMTD) != null ) {
+ isApply = true;
+ specFile = null;
+ applyTxPath = params.get(ParameterizedBuiltinFunctionExpression.TF_FN_PARAM_APPLYMTD);
+ }
+
+ if ( params.get(ParameterizedBuiltinFunctionExpression.TF_FN_PARAM_OUTNAMES) != null)
+ outNamesFile = params.get(ParameterizedBuiltinFunctionExpression.TF_FN_PARAM_OUTNAMES);
+
+ inputCSVProperties = (CSVFileFormatProperties)inputMatrix.getFileFormatProperties();
+ }
+
TransformOperands(ParameterizedBuiltinSPInstruction inst, MatrixObject inputMatrix) {
HashMap<String,String> params = inst.getParams();
@@ -1033,12 +1057,20 @@ public class DataTransform {
* @throws JSONException
* @throws IllegalArgumentException
*/
+ public static JobReturn cpDataTransform(ParameterizedBuiltinCPInstruction inst, MatrixObject[] inputMatrices, MatrixObject[] outputMatrices) throws IOException, DMLRuntimeException, IllegalArgumentException, JSONException {
+ TransformOperands oprnds = new TransformOperands(inst, inputMatrices[0]);
+ return cpDataTransform(oprnds, inputMatrices, outputMatrices);
+ }
+
public static JobReturn cpDataTransform(String inst, MatrixObject[] inputMatrices, MatrixObject[] outputMatrices) throws IOException, DMLRuntimeException, IllegalArgumentException, JSONException {
String[] insts = inst.split(Instruction.INSTRUCTION_DELIM);
-
// Parse transform instruction (the first instruction) to obtain relevant fields
TransformOperands oprnds = new TransformOperands(insts[0], inputMatrices[0]);
+ return cpDataTransform(oprnds, inputMatrices, outputMatrices);
+ }
+
+ public static JobReturn cpDataTransform(TransformOperands oprnds, MatrixObject[] inputMatrices, MatrixObject[] outputMatrices) throws IOException, DMLRuntimeException, IllegalArgumentException, JSONException {
JobConf job = new JobConf(ConfigurationManager.getCachedJobConf());
FileSystem fs = FileSystem.get(job);
// find the first file in alphabetical ordering of partfiles in directory inputPath
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/8040be60/src/test/java/com/ibm/bi/dml/test/integration/functions/transform/TransformTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/com/ibm/bi/dml/test/integration/functions/transform/TransformTest.java b/src/test/java/com/ibm/bi/dml/test/integration/functions/transform/TransformTest.java
index 23f8cbb..9c33fca 100644
--- a/src/test/java/com/ibm/bi/dml/test/integration/functions/transform/TransformTest.java
+++ b/src/test/java/com/ibm/bi/dml/test/integration/functions/transform/TransformTest.java
@@ -78,6 +78,12 @@ public class TransformTest extends AutomatedTestBase
}
@Test
+ public void testIrisSingleNodeCSV()
+ {
+ runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv", "iris", false);
+ }
+
+ @Test
public void testIrisSPHybridCSV()
{
runTransformTest(RUNTIME_PLATFORM.HYBRID_SPARK, "csv", "iris", false);
@@ -104,6 +110,12 @@ public class TransformTest extends AutomatedTestBase
}
@Test
+ public void testIrisSingleNodeBB()
+ {
+ runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "binary", "iris", false);
+ }
+
+ @Test
public void testIrisSPHybridBB()
{
runTransformTest(RUNTIME_PLATFORM.HYBRID_SPARK, "binary", "iris", false);
@@ -130,6 +142,12 @@ public class TransformTest extends AutomatedTestBase
}
@Test
+ public void testHomesSingleNodeCSV()
+ {
+ runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv", "homes", false);
+ }
+
+ @Test
public void testHomesHadoopCSV()
{
runTransformTest(RUNTIME_PLATFORM.HADOOP, "csv", "homes", false);
@@ -156,6 +174,12 @@ public class TransformTest extends AutomatedTestBase
}
@Test
+ public void testHomesSingleNodeBB()
+ {
+ runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "binary", "homes", false);
+ }
+
+ @Test
public void testHomesHadoopBB()
{
runTransformTest(RUNTIME_PLATFORM.HADOOP, "binary", "homes", false);
@@ -182,6 +206,12 @@ public class TransformTest extends AutomatedTestBase
}
@Test
+ public void testOmitHomesSingleNodeCSV()
+ {
+ runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv", "homesomit", false);
+ }
+
+ @Test
public void testOmitHomesHadoopCSV()
{
runTransformTest(RUNTIME_PLATFORM.HADOOP, "csv", "homesomit", false);
@@ -208,6 +238,12 @@ public class TransformTest extends AutomatedTestBase
}
@Test
+ public void testOmitHomesSingleNodeBB()
+ {
+ runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "binary", "homesomit", false);
+ }
+
+ @Test
public void testOmitHomesHadoopBB()
{
runTransformTest(RUNTIME_PLATFORM.HADOOP, "binary", "homesomit", false);
@@ -234,6 +270,12 @@ public class TransformTest extends AutomatedTestBase
}
@Test
+ public void testHomes2SingleNodeCSV()
+ {
+ runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv", "homes2", false);
+ }
+
+ @Test
public void testHomes2HadoopCSV()
{
runTransformTest(RUNTIME_PLATFORM.HADOOP, "csv", "homes2", false);
@@ -260,6 +302,12 @@ public class TransformTest extends AutomatedTestBase
}
@Test
+ public void testHomes2SingleNodeBB()
+ {
+ runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "binary", "homes2", false);
+ }
+
+ @Test
public void testHomes2HadoopBB()
{
runTransformTest(RUNTIME_PLATFORM.HADOOP, "binary", "homes2", false);
@@ -286,6 +334,12 @@ public class TransformTest extends AutomatedTestBase
}
@Test
+ public void testIrisSingleNodeIDCSV()
+ {
+ runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv", "iris", true);
+ }
+
+ @Test
public void testIrisSPHybridIDCSV()
{
runTransformTest(RUNTIME_PLATFORM.HYBRID_SPARK, "csv", "iris", true);
@@ -312,6 +366,12 @@ public class TransformTest extends AutomatedTestBase
}
@Test
+ public void testIrisSingleNodeIDBB()
+ {
+ runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "binary", "iris", true);
+ }
+
+ @Test
public void testIrisSPHybridIDBB()
{
runTransformTest(RUNTIME_PLATFORM.HYBRID_SPARK, "binary", "iris", true);
@@ -338,6 +398,12 @@ public class TransformTest extends AutomatedTestBase
}
@Test
+ public void testHomesSingleNodeIDCSV()
+ {
+ runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv", "homes", true);
+ }
+
+ @Test
public void testHomesSPHybridIDCSV()
{
runTransformTest(RUNTIME_PLATFORM.HYBRID_SPARK, "csv", "homes", true);
@@ -364,6 +430,12 @@ public class TransformTest extends AutomatedTestBase
}
@Test
+ public void testHomesSingleNodeIDBB()
+ {
+ runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "binary", "homes", true);
+ }
+
+ @Test
public void testHomesSPHybridIDBB()
{
runTransformTest(RUNTIME_PLATFORM.HYBRID_SPARK, "binary", "homes", true);
@@ -390,6 +462,12 @@ public class TransformTest extends AutomatedTestBase
}
@Test
+ public void testOmitHomesIDSingleNodeCSV()
+ {
+ runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv", "homesomit", true);
+ }
+
+ @Test
public void testOmitHomesIDHadoopCSV()
{
runTransformTest(RUNTIME_PLATFORM.HADOOP, "csv", "homesomit", true);
@@ -416,6 +494,12 @@ public class TransformTest extends AutomatedTestBase
}
@Test
+ public void testOmitHomesIDSingleNodeBB()
+ {
+ runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "binary", "homesomit", true);
+ }
+
+ @Test
public void testOmitHomesIDHadoopBB()
{
runTransformTest(RUNTIME_PLATFORM.HADOOP, "binary", "homesomit", true);
@@ -442,6 +526,12 @@ public class TransformTest extends AutomatedTestBase
}
@Test
+ public void testHomes2IDSingleNodeCSV()
+ {
+ runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv", "homes2", true);
+ }
+
+ @Test
public void testHomes2IDHadoopCSV()
{
runTransformTest(RUNTIME_PLATFORM.HADOOP, "csv", "homes2", true);
@@ -468,6 +558,12 @@ public class TransformTest extends AutomatedTestBase
}
@Test
+ public void testHomes2IDSingleNodeBB()
+ {
+ runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "binary", "homes2", true);
+ }
+
+ @Test
public void testHomes2IDHadoopBB()
{
runTransformTest(RUNTIME_PLATFORM.HADOOP, "binary", "homes2", true);
[3/5] incubator-systemml git commit: Fixing data gen constant value
check by taking spa rsity into account
Posted by sh...@apache.org.
Fixing data gen constant value check by taking spa rsity into account
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/c56b75a9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/c56b75a9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/c56b75a9
Branch: refs/heads/master
Commit: c56b75a90c87978a578abf0ced4e86c09b1cb6f9
Parents: 8ca83ab
Author: shirisht <sh...@gmail.com>
Authored: Wed Nov 25 01:04:00 2015 -0800
Committer: shirisht <sh...@gmail.com>
Committed: Wed Nov 25 01:04:00 2015 -0800
----------------------------------------------------------------------
.../java/com/ibm/bi/dml/hops/DataGenOp.java | 20 ++++++++++++++++----
1 file changed, 16 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c56b75a9/src/main/java/com/ibm/bi/dml/hops/DataGenOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/hops/DataGenOp.java b/src/main/java/com/ibm/bi/dml/hops/DataGenOp.java
index 6227b53..90d5ec0 100644
--- a/src/main/java/com/ibm/bi/dml/hops/DataGenOp.java
+++ b/src/main/java/com/ibm/bi/dml/hops/DataGenOp.java
@@ -393,22 +393,34 @@ public class DataGenOp extends Hop implements MultiThreadedHop
Hop min = getInput().get(_paramIndexMap.get(DataExpression.RAND_MIN)); //min
Hop max = getInput().get(_paramIndexMap.get(DataExpression.RAND_MAX)); //max
+ Hop sparsity = getInput().get(_paramIndexMap.get(DataExpression.RAND_SPARSITY)); //sparsity
//literal value comparison
- if( min instanceof LiteralOp && max instanceof LiteralOp){
+ if( min instanceof LiteralOp && max instanceof LiteralOp && sparsity instanceof LiteralOp){
try{
double minVal = HopRewriteUtils.getDoubleValue((LiteralOp)min);
double maxVal = HopRewriteUtils.getDoubleValue((LiteralOp)max);
- return (minVal == maxVal);
+ double sp = HopRewriteUtils.getDoubleValue((LiteralOp)sparsity);
+ return (sp==1.0 && minVal == maxVal);
}
catch(Exception ex)
{
return false;
}
}
-
//reference comparison (based on common subexpression elimination)
- return (min == max);
+ else if ( min == max && sparsity instanceof LiteralOp ) {
+ try {
+ double sp = HopRewriteUtils.getDoubleValue((LiteralOp)sparsity);
+ return (sp==1.0);
+ }
+ catch(Exception ex)
+ {
+ return false;
+ }
+ }
+
+ return false;
}
/**
[4/5] incubator-systemml git commit: Fixing typo (package name) in
UDF wrapper classes
Posted by sh...@apache.org.
Fixing typo (package name) in UDF wrapper classes
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/c4b01a26
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/c4b01a26
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/c4b01a26
Branch: refs/heads/master
Commit: c4b01a263c9f097ca11a424827467f0e6f9d8265
Parents: c56b75a
Author: shirisht <sh...@gmail.com>
Authored: Wed Nov 25 01:06:06 2015 -0800
Committer: shirisht <sh...@gmail.com>
Committed: Wed Nov 25 01:06:06 2015 -0800
----------------------------------------------------------------------
src/main/java/com/ibm/bi/dml/udf/lib/PermutationMatrixWrapper.java | 2 +-
src/main/java/com/ibm/bi/dml/udf/lib/TimeWrapper.java | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c4b01a26/src/main/java/com/ibm/bi/dml/udf/lib/PermutationMatrixWrapper.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/udf/lib/PermutationMatrixWrapper.java b/src/main/java/com/ibm/bi/dml/udf/lib/PermutationMatrixWrapper.java
index be11343..9561d7e 100644
--- a/src/main/java/com/ibm/bi/dml/udf/lib/PermutationMatrixWrapper.java
+++ b/src/main/java/com/ibm/bi/dml/udf/lib/PermutationMatrixWrapper.java
@@ -43,7 +43,7 @@ import com.ibm.bi.dml.udf.Matrix.ValueType;
*
* permutation_matrix= externalFunction(Matrix[Double] A, Boolean desc) return
* (Matrix[Double] P) implemented in
- * (classname="com.ibm.bi.dml.packagesupport.PermutationMatrixWrapper"
+ * (classname="com.ibm.bi.dml.udf.lib.PermutationMatrixWrapper"
* ,exectype="mem"); A = read( "Data/A.mtx"); P = permutation_matrix( A[,2],
* false); B = P %*% A
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c4b01a26/src/main/java/com/ibm/bi/dml/udf/lib/TimeWrapper.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/udf/lib/TimeWrapper.java b/src/main/java/com/ibm/bi/dml/udf/lib/TimeWrapper.java
index 7d67ca8..0eb181a 100644
--- a/src/main/java/com/ibm/bi/dml/udf/lib/TimeWrapper.java
+++ b/src/main/java/com/ibm/bi/dml/udf/lib/TimeWrapper.java
@@ -30,7 +30,7 @@ import com.ibm.bi.dml.udf.Scalar.ScalarValueType;
* Wrapper class for time invocation
*
* time = externalFunction(Integer i) return (Double B) implemented in
- * (classname="com.ibm.bi.dml.packagesupport.TimeWrapper",exectype="mem");
+ * (classname="com.ibm.bi.dml.udf.lib.TimeWrapper",exectype="mem");
*
* t = time (1);
*
[5/5] incubator-systemml git commit: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/incubator-systemml.git
Posted by sh...@apache.org.
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/incubator-systemml.git
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/5c88842d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/5c88842d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/5c88842d
Branch: refs/heads/master
Commit: 5c88842d12b48ee959c50bc93fcd378323706fb6
Parents: c4b01a2 52ec04a
Author: shirisht <sh...@gmail.com>
Authored: Wed Nov 25 14:34:01 2015 -0800
Committer: shirisht <sh...@gmail.com>
Committed: Wed Nov 25 14:34:01 2015 -0800
----------------------------------------------------------------------
.../java/com/ibm/bi/dml/hops/AggBinaryOp.java | 7 +++--
.../parfor/opt/OptimizerRuleBased.java | 21 ++++++++++-----
.../dml/runtime/matrix/data/LibMatrixReorg.java | 27 ++++++++++++--------
.../ibm/bi/dml/runtime/util/DataConverter.java | 25 ++++++++----------
4 files changed, 45 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
[2/5] incubator-systemml git commit: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/incubator-systemml.git
Posted by sh...@apache.org.
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/incubator-systemml.git
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/8ca83ab6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/8ca83ab6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/8ca83ab6
Branch: refs/heads/master
Commit: 8ca83ab6b573cadf65e2c5ee3c6d61fe8028ac24
Parents: 8040be6 ff0bdad
Author: shirisht <st...@us.ibm.com>
Authored: Wed Nov 25 00:43:26 2015 -0800
Committer: shirisht <st...@us.ibm.com>
Committed: Wed Nov 25 00:43:26 2015 -0800
----------------------------------------------------------------------
.../dml/runtime/matrix/data/LibMatrixMult.java | 35 ++++++++++++++++++--
.../functions/aggregate/ZPackageSuite.java | 3 ++
2 files changed, 36 insertions(+), 2 deletions(-)
----------------------------------------------------------------------