You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2017/09/13 03:13:23 UTC
[2/2] systemml git commit: [SYSTEMML-1905] Extended codegen plan
enumeration statistics (all plans)
[SYSTEMML-1905] Extended codegen plan enumeration statistics (all plans)
This patch extends the codegen plan enumeration statistics by the number
of total plans without partitioning by connected components, i.e., the
sum of 2^(|M|) per DAG.
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/3acd9418
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/3acd9418
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/3acd9418
Branch: refs/heads/master
Commit: 3acd94186b5e6e2fdf12823e7932bf56027511c6
Parents: 808a8f4
Author: Matthias Boehm <mb...@gmail.com>
Authored: Tue Sep 12 19:10:58 2017 -0700
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Tue Sep 12 20:13:18 2017 -0700
----------------------------------------------------------------------
.../hops/codegen/opt/PlanSelectionFuseCostBased.java | 9 ++++++++-
.../hops/codegen/opt/PlanSelectionFuseCostBasedV2.java | 8 +++++++-
src/main/java/org/apache/sysml/utils/Statistics.java | 12 ++++++++++--
3 files changed, 25 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/3acd9418/src/main/java/org/apache/sysml/hops/codegen/opt/PlanSelectionFuseCostBased.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/codegen/opt/PlanSelectionFuseCostBased.java b/src/main/java/org/apache/sysml/hops/codegen/opt/PlanSelectionFuseCostBased.java
index 4fcef6e..c230505 100644
--- a/src/main/java/org/apache/sysml/hops/codegen/opt/PlanSelectionFuseCostBased.java
+++ b/src/main/java/org/apache/sysml/hops/codegen/opt/PlanSelectionFuseCostBased.java
@@ -57,6 +57,7 @@ import org.apache.sysml.hops.codegen.template.TemplateBase.TemplateType;
import org.apache.sysml.hops.rewrite.HopRewriteUtils;
import org.apache.sysml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
import org.apache.sysml.runtime.controlprogram.parfor.util.IDSequence;
+import org.apache.sysml.runtime.util.UtilFunctions;
import org.apache.sysml.utils.Statistics;
/**
@@ -87,12 +88,14 @@ public class PlanSelectionFuseCostBased extends PlanSelection
Collection<PlanPartition> parts = PlanAnalyzer.analyzePlanPartitions(memo, roots, false);
//step 2: optimize individual plan partitions
+ int sumMatPoints = 0;
for( PlanPartition part : parts ) {
//create composite templates (within the partition)
createAndAddMultiAggPlans(memo, part.getPartition(), part.getRoots());
//plan enumeration and plan selection
selectPlans(memo, part.getPartition(), part.getRoots(), part.getMatPoints());
+ sumMatPoints += part.getMatPoints().size();
}
//step 3: add composite templates (across partitions)
@@ -101,6 +104,10 @@ public class PlanSelectionFuseCostBased extends PlanSelection
//take all distinct best plans
for( Entry<Long, List<MemoTableEntry>> e : getBestPlans().entrySet() )
memo.setDistinct(e.getKey(), e.getValue());
+
+ //maintain statistics
+ if( DMLScript.STATISTICS )
+ Statistics.incrementCodegenEnumAll(UtilFunctions.pow(2, sumMatPoints));
}
//within-partition multi-agg templates
@@ -389,7 +396,7 @@ public class PlanSelectionFuseCostBased extends PlanSelection
}
if( DMLScript.STATISTICS ) {
- Statistics.incrementCodegenEnumAll(len);
+ Statistics.incrementCodegenEnumAllP(len);
Statistics.incrementCodegenEnumEval(len);
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/3acd9418/src/main/java/org/apache/sysml/hops/codegen/opt/PlanSelectionFuseCostBasedV2.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/codegen/opt/PlanSelectionFuseCostBasedV2.java b/src/main/java/org/apache/sysml/hops/codegen/opt/PlanSelectionFuseCostBasedV2.java
index a8f7365..1bf42f6 100644
--- a/src/main/java/org/apache/sysml/hops/codegen/opt/PlanSelectionFuseCostBasedV2.java
+++ b/src/main/java/org/apache/sysml/hops/codegen/opt/PlanSelectionFuseCostBasedV2.java
@@ -107,12 +107,14 @@ public class PlanSelectionFuseCostBasedV2 extends PlanSelection
Collection<PlanPartition> parts = PlanAnalyzer.analyzePlanPartitions(memo, roots, true);
//step 2: optimize individual plan partitions
+ int sumMatPoints = 0;
for( PlanPartition part : parts ) {
//create composite templates (within the partition)
createAndAddMultiAggPlans(memo, part.getPartition(), part.getRoots());
//plan enumeration and plan selection
selectPlans(memo, part);
+ sumMatPoints += part.getMatPointsExt().length;
}
//step 3: add composite templates (across partitions)
@@ -121,6 +123,10 @@ public class PlanSelectionFuseCostBasedV2 extends PlanSelection
//take all distinct best plans
for( Entry<Long, List<MemoTableEntry>> e : getBestPlans().entrySet() )
memo.setDistinct(e.getKey(), e.getValue());
+
+ //maintain statistics
+ if( DMLScript.STATISTICS )
+ Statistics.incrementCodegenEnumAll(UtilFunctions.pow(2, sumMatPoints));
}
private void selectPlans(CPlanMemoTable memo, PlanPartition part)
@@ -257,7 +263,7 @@ public class PlanSelectionFuseCostBasedV2 extends PlanSelection
}
if( DMLScript.STATISTICS ) {
- Statistics.incrementCodegenEnumAll((rgraph!=null)?len:0);
+ Statistics.incrementCodegenEnumAllP((rgraph!=null)?len:0);
Statistics.incrementCodegenEnumEval(numEvalPlans);
Statistics.incrementCodegenEnumEvalP(numEvalPartPlans);
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/3acd9418/src/main/java/org/apache/sysml/utils/Statistics.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/utils/Statistics.java b/src/main/java/org/apache/sysml/utils/Statistics.java
index bab668f..109eec2 100644
--- a/src/main/java/org/apache/sysml/utils/Statistics.java
+++ b/src/main/java/org/apache/sysml/utils/Statistics.java
@@ -79,6 +79,7 @@ public class Statistics
private static final LongAdder codegenCPlanCompile = new LongAdder(); //count
private static final LongAdder codegenClassCompile = new LongAdder(); //count
private static final LongAdder codegenEnumAll = new LongAdder(); //count
+ private static final LongAdder codegenEnumAllP = new LongAdder(); //count
private static final LongAdder codegenEnumEval = new LongAdder(); //count
private static final LongAdder codegenEnumEvalP = new LongAdder(); //count
private static final LongAdder codegenPlanCacheHits = new LongAdder(); //count
@@ -262,6 +263,9 @@ public class Statistics
public static void incrementCodegenEnumAll(long delta) {
codegenEnumAll.add(delta);
}
+ public static void incrementCodegenEnumAllP(long delta) {
+ codegenEnumAllP.add(delta);
+ }
public static void incrementCodegenEnumEval(long delta) {
codegenEnumEval.add(delta);
}
@@ -300,6 +304,9 @@ public class Statistics
public static long getCodegenEnumAll() {
return codegenEnumAll.longValue();
}
+ public static long getCodegenEnumAllP() {
+ return codegenEnumAllP.longValue();
+ }
public static long getCodegenEnumEval() {
return codegenEnumEval.longValue();
}
@@ -403,6 +410,7 @@ public class Statistics
codegenCPlanCompile.reset();
codegenClassCompile.reset();
codegenEnumAll.reset();
+ codegenEnumAllP.reset();
codegenEnumEval.reset();
codegenEnumEvalP.reset();
codegenCompileTime.reset();
@@ -795,8 +803,8 @@ public class Statistics
if( ConfigurationManager.isCodegenEnabled() ) {
sb.append("Codegen compile (DAG,CP,JC):\t" + getCodegenDAGCompile() + "/"
+ getCodegenCPlanCompile() + "/" + getCodegenClassCompile() + ".\n");
- sb.append("Codegen enum (All,Eval,EvalP):\t" + getCodegenEnumAll() + "/"
- + getCodegenEnumEval() + "/" + getCodegenEnumEvalP() + ".\n");
+ sb.append("Codegen enum (ALLt/p,EVALt/p):\t" + getCodegenEnumAll() + "/" +
+ getCodegenEnumAllP() + "/" + getCodegenEnumEval() + "/" + getCodegenEnumEvalP() + ".\n");
sb.append("Codegen compile times (DAG,JC):\t" + String.format("%.3f", (double)getCodegenCompileTime()/1000000000) + "/" +
String.format("%.3f", (double)getCodegenClassCompileTime()/1000000000) + " sec.\n");
sb.append("Codegen plan cache hits:\t" + getCodegenPlanCacheHits() + "/" + getCodegenPlanCacheTotal() + ".\n");