You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2015/12/19 19:28:02 UTC
[4/4] incubator-systemml git commit: Fix grouped aggregate output
size, incl extended tests (caching, meta)
Fix grouped aggregate output size, incl extended tests (caching, meta)
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/7290510e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/7290510e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/7290510e
Branch: refs/heads/master
Commit: 7290510ec26ac2949b0d21d31eac768726929550
Parents: 112ba90
Author: Matthias Boehm <mb...@us.ibm.com>
Authored: Sat Dec 19 19:26:39 2015 +0100
Committer: Matthias Boehm <mb...@us.ibm.com>
Committed: Sat Dec 19 19:26:39 2015 +0100
----------------------------------------------------------------------
.../apache/sysml/hops/ParameterizedBuiltinOp.java | 15 ++++++---------
.../sysml/runtime/matrix/data/MatrixBlock.java | 4 ++--
.../sysml/test/integration/applications/ID3Test.java | 2 +-
.../aggregate/FullGroupedAggregateMatrixTest.java | 11 +++++++----
4 files changed, 16 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7290510e/src/main/java/org/apache/sysml/hops/ParameterizedBuiltinOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/ParameterizedBuiltinOp.java b/src/main/java/org/apache/sysml/hops/ParameterizedBuiltinOp.java
index 3a0a4f5..51d4ba7 100644
--- a/src/main/java/org/apache/sysml/hops/ParameterizedBuiltinOp.java
+++ b/src/main/java/org/apache/sysml/hops/ParameterizedBuiltinOp.java
@@ -242,7 +242,8 @@ public class ParameterizedBuiltinOp extends Hop
Hop groups = getInput().get(_paramIndexMap.get(Statement.GAGG_GROUPS));
Lop append = null;
- if( target.getDim2()>=target.getColsInBlock() ) //multi-column-block result matrix
+ if( target.getDim2()>=target.getColsInBlock() // multi-column-block result matrix
+ || target.getDim2()<=0 ) // unkown
{
long m1_dim1 = target.getDim1();
long m1_dim2 = target.getDim2();
@@ -273,10 +274,8 @@ public class ParameterizedBuiltinOp extends Hop
}
else //single-column-block vector or matrix
{
- append = BinaryOp.constructMRAppendLop(
- target, groups,
- DataType.MATRIX, getValueType(), true,
- getInput().get(_paramIndexMap.get(Statement.GAGG_TARGET)));
+ append = BinaryOp.constructMRAppendLop(target, groups,
+ DataType.MATRIX, getValueType(), true, target);
}
// add the combine lop to parameter list, with a new name "combinedinput"
@@ -307,11 +306,9 @@ public class ParameterizedBuiltinOp extends Hop
}
GroupedAggregate grp_agg = new GroupedAggregate(inputlops, isWeighted, getDataType(), getValueType());
-
- // output dimensions are unknown at compilation time
grp_agg.getOutputParameters().setDimensions(outputDim1, outputDim2, getRowsInBlock(), getColsInBlock(), -1);
- grp_agg.setAllPositions(this.getBeginLine(), this.getBeginColumn(), this.getEndLine(), this.getEndColumn());
-
+ setLineNumbers(grp_agg);
+
setLops(grp_agg);
setRequiresReblock( true );
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7290510e/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java
index 0fe9095..ca569f3 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java
@@ -5328,9 +5328,9 @@ public class MatrixBlock extends MatrixValue implements Externalizable
MatrixBlock result = checkType(ret);
boolean result_sparsity = estimateSparsityOnGroupedAgg(rlen, numGroups);
if(result==null)
- result=new MatrixBlock(numGroups, rowVector?1:target.getNumRows(), result_sparsity);
+ result=new MatrixBlock(numGroups, rowVector?1:target.getNumColumns(), result_sparsity);
else
- result.reset(numGroups, rowVector?1:target.getNumRows(), result_sparsity);
+ result.reset(numGroups, rowVector?1:target.getNumColumns(), result_sparsity);
//CM operator for count, mean, variance
//note: current support only for column vectors
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7290510e/src/test/java/org/apache/sysml/test/integration/applications/ID3Test.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/applications/ID3Test.java b/src/test/java/org/apache/sysml/test/integration/applications/ID3Test.java
index e80ce29..c7d3cb4 100644
--- a/src/test/java/org/apache/sysml/test/integration/applications/ID3Test.java
+++ b/src/test/java/org/apache/sysml/test/integration/applications/ID3Test.java
@@ -97,7 +97,7 @@ public abstract class ID3Test extends AutomatedTestBase
//run tests
//(changed expected MR from 62 to 66 because we now also count MR jobs in predicates)
//(changed expected MR from 66 to 68 because we now rewrite sum(v1*v2) to t(v1)%*%v2 which rarely creates more jobs due to MMCJ incompatibility of other operations)
- runTest(true, EXCEPTION_NOT_EXPECTED, null, 68); //max 68 compiled jobs
+ runTest(true, EXCEPTION_NOT_EXPECTED, null, 70); //max 68 compiled jobs
runRScript(true);
//check also num actually executed jobs
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7290510e/src/test/java/org/apache/sysml/test/integration/functions/aggregate/FullGroupedAggregateMatrixTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/aggregate/FullGroupedAggregateMatrixTest.java b/src/test/java/org/apache/sysml/test/integration/functions/aggregate/FullGroupedAggregateMatrixTest.java
index ec1f989..6a165f2 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/aggregate/FullGroupedAggregateMatrixTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/aggregate/FullGroupedAggregateMatrixTest.java
@@ -349,7 +349,7 @@ public class FullGroupedAggregateMatrixTest extends AutomatedTestBase
String TEST_NAME = testname;
int fn = type.ordinal();
double sparsity = (sparse) ? sparsity1 : sparsity2;
- String TEST_CACHE_DIR = TEST_CACHE_ENABLED ? TEST_NAME + type.ordinal() + "_" + sparsity + "/" : "";
+ String TEST_CACHE_DIR = TEST_CACHE_ENABLED ? TEST_NAME + type.ordinal() + "_" + sparsity + "_" + numCols + "/" : "";
boolean exceptionExpected = !TEST_NAME.equals(TEST_NAME1);
TestConfiguration config = getTestConfiguration(TEST_NAME);
@@ -379,11 +379,16 @@ public class FullGroupedAggregateMatrixTest extends AutomatedTestBase
//compare matrices
if( !exceptionExpected ){
+ //run R script for comparison
runRScript(true);
+ //compare output matrices
HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS("C");
HashMap<CellIndex, Double> rfile = readRMatrixFromFS("C");
TestUtils.compareMatrices(dmlfile, rfile, eps, "Stat-DML", "Stat-R");
+
+ //check dml output meta data
+ checkDMLMetaDataFile("C", new MatrixCharacteristics(numGroups,numCols,1,1));
}
}
catch(IOException ex)
@@ -397,6 +402,4 @@ public class FullGroupedAggregateMatrixTest extends AutomatedTestBase
DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
}
}
-
-
-}
\ No newline at end of file
+}