You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2017/10/07 23:40:51 UTC
systemml git commit: [SYSTEMML-1946] Fix constrained parfor optimizer
w/ fixed parallelism
Repository: systemml
Updated Branches:
refs/heads/master 4e0c7f1c9 -> bfb30b3af
[SYSTEMML-1946] Fix constrained parfor optimizer w/ fixed parallelism
This patch fixes the constrained parfor optimizer for the case of a
specified parfor degree of parallelism to properly assign the remaining
parallelism to nested parfor loops and operations. So far all operations
ran at maximum parallelism leading to large CPU over-provisioning, which
can severely hurt performance.
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/bfb30b3a
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/bfb30b3a
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/bfb30b3a
Branch: refs/heads/master
Commit: bfb30b3afaa0f88b946e326799cda81754e7bcab
Parents: 4e0c7f1
Author: Matthias Boehm <mb...@gmail.com>
Authored: Sat Oct 7 16:41:02 2017 -0700
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Sat Oct 7 16:41:02 2017 -0700
----------------------------------------------------------------------
.../controlprogram/parfor/opt/OptimizerConstrained.java | 7 +++++++
.../runtime/controlprogram/parfor/opt/OptimizerRuleBased.java | 6 +++---
2 files changed, 10 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/bfb30b3a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerConstrained.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerConstrained.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerConstrained.java
index 1ca5631..d571a71 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerConstrained.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerConstrained.java
@@ -293,9 +293,16 @@ public class OptimizerConstrained extends OptimizerRuleBased
// constraint awareness
if( n.getK() > 0 && ConfigurationManager.isParallelParFor() )
{
+ //set parfor degree of parallelism
ParForProgramBlock pfpb = (ParForProgramBlock) OptTreeConverter
.getAbstractPlanMapping().getMappedProg(n.getID())[1];
pfpb.setDegreeOfParallelism(n.getK());
+
+ //distribute remaining parallelism
+ int remainParforK = getRemainingParallelismParFor(n.getK(), n.getK());
+ int remainOpsK = getRemainingParallelismOps(_lkmaxCP, n.getK());
+ rAssignRemainingParallelism( n, remainParforK, remainOpsK );
+
LOG.debug(getOptMode()+" OPT: forced 'set degree of parallelism' - result=(see EXPLAIN)" );
}
else
http://git-wip-us.apache.org/repos/asf/systemml/blob/bfb30b3a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java
index b80e786..e035f1b 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java
@@ -1258,7 +1258,7 @@ public class OptimizerRuleBased extends Optimizer
//distribute remaining parallelism
int remainParforK = getRemainingParallelismParFor(kMax, parforK);
int remainOpsK = getRemainingParallelismOps(_lkmaxCP, parforK);
- rAssignRemainingParallelism( n, remainParforK, remainOpsK );
+ rAssignRemainingParallelism( n, remainParforK, remainOpsK );
}
else // ExecType.MR/ExecType.SPARK
{
@@ -1373,12 +1373,12 @@ public class OptimizerRuleBased extends Optimizer
}
}
- private static int getRemainingParallelismParFor(int parforK, int tmpK) {
+ protected static int getRemainingParallelismParFor(int parforK, int tmpK) {
//compute max remaining parfor parallelism k such that k * tmpK <= parforK
return (int)Math.ceil((double)(parforK-tmpK+1) / tmpK);
}
- private static int getRemainingParallelismOps(int opsK, int tmpK) {
+ protected static int getRemainingParallelismOps(int opsK, int tmpK) {
//compute max remaining operations parallelism k with slight over-provisioning
//such that k * tmpK <= 1.5 * opsK; note that if parfor already exploits the
//maximum parallelism, this will not introduce any over-provisioning.