You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2018/05/31 05:05:33 UTC
[1/2] systemml git commit: [MINOR] Fix consistency explain/stats
output to stdout (instead of log)
Repository: systemml
Updated Branches:
refs/heads/master 7350a0c6d -> af9cc8a90
[MINOR] Fix consistency explain/stats output to stdout (instead of log)
This patch redirects the -explain output to stdout instead of LOG.info
for consistency with the -stats output. On recent HDP versions the log
level is by default set to WARN resulting in the explain output not
being shown. This led to confusion because the stats output was handled
differently.
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/7eebb42e
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/7eebb42e
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/7eebb42e
Branch: refs/heads/master
Commit: 7eebb42ea778e5e313d3a7fad2af6d674077d6a9
Parents: 7350a0c
Author: Matthias Boehm <mb...@gmail.com>
Authored: Wed May 30 21:53:06 2018 -0700
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Wed May 30 21:53:06 2018 -0700
----------------------------------------------------------------------
src/main/java/org/apache/sysml/api/DMLScript.java | 10 +++++-----
.../apache/sysml/runtime/instructions/cp/ListObject.java | 2 +-
2 files changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/7eebb42e/src/main/java/org/apache/sysml/api/DMLScript.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/DMLScript.java b/src/main/java/org/apache/sysml/api/DMLScript.java
index c8c2e39..07e766a 100644
--- a/src/main/java/org/apache/sysml/api/DMLScript.java
+++ b/src/main/java/org/apache/sysml/api/DMLScript.java
@@ -678,8 +678,8 @@ public class DMLScript
try {
DMLScript.GPU_EVICTION_POLICY = EvictionPolicy.valueOf(evictionPolicy);
} catch(IllegalArgumentException e) {
- throw new RuntimeException("Unsupported eviction policy:" + evictionPolicy);
- }
+ throw new RuntimeException("Unsupported eviction policy:" + evictionPolicy);
+ }
//Step 2: set local/remote memory if requested (for compile in AM context)
if( dmlconf.getBooleanValue(DMLConfig.YARN_APPMASTER) ){
@@ -726,11 +726,11 @@ public class DMLScript
//Step 9: prepare statistics [and optional explain output]
//count number compiled MR jobs / SP instructions
ExplainCounts counts = Explain.countDistributedOperations(rtprog);
- Statistics.resetNoOfCompiledJobs( counts.numJobs );
+ Statistics.resetNoOfCompiledJobs( counts.numJobs );
//explain plan of program (hops or runtime)
if( EXPLAIN != ExplainType.NONE )
- LOG.info(Explain.display(prog, rtprog, EXPLAIN, counts));
+ System.out.println(Explain.display(prog, rtprog, EXPLAIN, counts));
Statistics.stopCompileTimer();
@@ -750,7 +750,7 @@ public class DMLScript
//cleanup scratch_space and all working dirs
cleanupHadoopExecution( dmlconf );
}
- }
+ }
/**
* Launcher for DML debugger. This method should be called after
http://git-wip-us.apache.org/repos/asf/systemml/blob/7eebb42e/src/main/java/org/apache/sysml/runtime/instructions/cp/ListObject.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/ListObject.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/ListObject.java
index 670190c..4c06396 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/ListObject.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/ListObject.java
@@ -118,7 +118,7 @@ public class ListObject extends Data {
public long getDataSize() {
return _data.stream().filter(data -> data instanceof CacheableData)
- .map(data -> ((CacheableData) data).getDataSize()).reduce((l1, l2) -> l1 + l2).get();
+ .mapToLong(data -> ((CacheableData<?>) data).getDataSize()).sum();
}
@Override
[2/2] systemml git commit: [SYSTEMML-2351] Performance sparse
matrix-vector mult w/ large vector
Posted by mb...@apache.org.
[SYSTEMML-2351] Performance sparse matrix-vector mult w/ large vector
This patch improves the performance of sparse matrix-vector
multiplication with large vectors (> L3 cache size) as used for
algorithms such as PageRank and local singlenode operations. Although we
already had a cache-conscious implementation, this case required
additional tuning. In detail, this includes better blocking for L1 (pos
and output vector), and L2 (input vector) which allows a better
exploitation of cache locality of the vector across rows of the large,
sparse input matrix.
For executing 100 iterations of PageRank over a graph with 10M nodes and
5G edges (~60GB, 80MB vector) on a 2-socket box w/ Xeon Gold 6138 (80
vcores), this patch improved the end-to-end runtime from 288s to 164s.
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/af9cc8a9
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/af9cc8a9
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/af9cc8a9
Branch: refs/heads/master
Commit: af9cc8a90b73520b04ecd579d7359a62d577009f
Parents: 7eebb42
Author: Matthias Boehm <mb...@gmail.com>
Authored: Wed May 30 22:05:18 2018 -0700
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Wed May 30 22:05:18 2018 -0700
----------------------------------------------------------------------
.../runtime/matrix/data/LibMatrixMult.java | 23 +++++++++++++++++---
1 file changed, 20 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/af9cc8a9/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixMult.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixMult.java b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixMult.java
index 92a6b7a..31f827b 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixMult.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixMult.java
@@ -1237,14 +1237,14 @@ public class LibMatrixMult
double[] bvals = b.valuesAt(0);
double[] cvals = c.valuesAt(0);
- final int blocksizeI = 32;
- final int blocksizeK = (int)Math.max(2*1024,2*1024*xsp/32); //~ 16KB L1
+ final int blocksizeI = 512; //8KB curk+cvals in L1
+ final int blocksizeK = (int)Math.max(2048,2048*xsp/32); //~256KB bvals in L2
int[] curk = new int[blocksizeI];
for( int bi = rl; bi < ru; bi+=blocksizeI ) {
Arrays.fill(curk, 0); //reset positions
for( int bk=0, bimin = Math.min(ru, bi+blocksizeI); bk<cd; bk+=blocksizeK ) {
- for( int i=bi, bkmin = Math.min(bk+blocksizeK, cd); i<bimin; i++) {
+ for( int i=bi, bkmin = bk+blocksizeK; i<bimin; i++) {
if( a.isEmpty(i) ) continue;
int apos = a.pos(i);
int alen = a.size(i);
@@ -3814,6 +3814,23 @@ public class LibMatrixMult
return knnz;
}
+
+ @SuppressWarnings("unused")
+ private static void resetPosVect(int[] curk, SparseBlock sblock, int rl, int ru) {
+ if( sblock instanceof SparseBlockMCSR ) {
+ //all rows start at position 0 (individual arrays)
+ Arrays.fill(curk, 0, ru-rl, 0);
+ }
+ else if( sblock instanceof SparseBlockCSR ) {
+ //row start positions given in row ptr array
+ SparseBlockCSR csr = (SparseBlockCSR) sblock;
+ System.arraycopy(csr.rowPointers(), rl, curk, 0, ru-rl);
+ }
+ else { //general case
+ for(int i=rl; i<ru; i++)
+ curk[i-rl] = sblock.pos(i);
+ }
+ }
private static void sumScalarResults(List<Future<Double>> tasks, MatrixBlock ret)
throws InterruptedException, ExecutionException