You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2017/08/25 19:23:55 UTC
[1/2] systemml git commit: [SYSTEMML-1871] Rework compiler/runtime
predicate handling (if, loops)
Repository: systemml
Updated Branches:
refs/heads/master 100f2d606 -> 1d83cedb7
http://git-wip-us.apache.org/repos/asf/systemml/blob/1d83cedb/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ProgramConverter.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ProgramConverter.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ProgramConverter.java
index afdf8be..8dc86d4 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ProgramConverter.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ProgramConverter.java
@@ -268,7 +268,6 @@ public class ProgramConverter
{
ArrayList<Instruction> predinst = createDeepCopyInstructionSet(wpb.getPredicate(), pid, IDPrefix, prog, fnStack, fnCreated, plain, true);
WhileProgramBlock tmpPB = new WhileProgramBlock(prog, predinst);
- tmpPB.setPredicateResultVar( wpb.getPredicateResultVar() );
tmpPB.setStatementBlock( createWhileStatementBlockCopy((WhileStatementBlock) wpb.getStatementBlock(), pid, plain, forceDeepCopy) );
tmpPB.setThreadID(pid);
@@ -283,7 +282,6 @@ public class ProgramConverter
{
ArrayList<Instruction> predinst = createDeepCopyInstructionSet(ipb.getPredicate(), pid, IDPrefix, prog, fnStack, fnCreated, plain, true);
IfProgramBlock tmpPB = new IfProgramBlock(prog, predinst);
- tmpPB.setPredicateResultVar( ipb.getPredicateResultVar() );
tmpPB.setStatementBlock( createIfStatementBlockCopy((IfStatementBlock)ipb.getStatementBlock(), pid, plain, forceDeepCopy ) );
tmpPB.setThreadID(pid);
@@ -297,7 +295,7 @@ public class ProgramConverter
public static ForProgramBlock createDeepCopyForProgramBlock(ForProgramBlock fpb, long pid, int IDPrefix, Program prog, HashSet<String> fnStack, HashSet<String> fnCreated, boolean plain, boolean forceDeepCopy)
throws DMLRuntimeException
{
- ForProgramBlock tmpPB = new ForProgramBlock(prog,fpb.getIterablePredicateVars());
+ ForProgramBlock tmpPB = new ForProgramBlock(prog,fpb.getIterVar());
tmpPB.setStatementBlock( createForStatementBlockCopy((ForStatementBlock)fpb.getStatementBlock(), pid, plain, forceDeepCopy));
tmpPB.setThreadID(pid);
@@ -313,7 +311,7 @@ public class ProgramConverter
public static ForProgramBlock createShallowCopyForProgramBlock(ForProgramBlock fpb, Program prog )
throws DMLRuntimeException
{
- ForProgramBlock tmpPB = new ForProgramBlock(prog,fpb.getIterablePredicateVars());
+ ForProgramBlock tmpPB = new ForProgramBlock(prog,fpb.getIterVar());
tmpPB.setFromInstructions( fpb.getFromInstructions() );
tmpPB.setToInstructions( fpb.getToInstructions() );
@@ -330,9 +328,9 @@ public class ProgramConverter
ParForProgramBlock tmpPB = null;
if( IDPrefix == -1 ) //still on master node
- tmpPB = new ParForProgramBlock(prog,pfpb.getIterablePredicateVars(),pfpb.getParForParams());
+ tmpPB = new ParForProgramBlock(prog,pfpb.getIterVar(), pfpb.getParForParams());
else //child of remote ParWorker at any level
- tmpPB = new ParForProgramBlock(IDPrefix, prog, pfpb.getIterablePredicateVars(),pfpb.getParForParams());
+ tmpPB = new ParForProgramBlock(IDPrefix, prog, pfpb.getIterVar(), pfpb.getParForParams());
tmpPB.setStatementBlock( createForStatementBlockCopy( (ForStatementBlock) pfpb.getStatementBlock(), pid, plain, forceDeepCopy) );
tmpPB.setThreadID(pid);
@@ -584,7 +582,7 @@ public class ProgramConverter
try
{
- if( ConfigurationManager.getCompilerConfigFlag(ConfigType.ALLOW_PARALLEL_DYN_RECOMPILATION)
+ if( ConfigurationManager.getCompilerConfigFlag(ConfigType.ALLOW_PARALLEL_DYN_RECOMPILATION)
&& sb != null //forced deep copy for function recompile
&& (Recompiler.requiresRecompilation( sb.getPredicateHops() ) || forceDeepCopy) )
{
@@ -729,8 +727,8 @@ public class ProgramConverter
throws DMLRuntimeException
{
ArrayList<ProgramBlock> pbs = body.getChildBlocks();
- ArrayList<String> rVnames = body.getResultVarNames();
- ExecutionContext ec = body.getEc();
+ ArrayList<String> rVnames = body.getResultVarNames();
+ ExecutionContext ec = body.getEc();
if( pbs.isEmpty() )
return PARFORBODY_BEGIN + PARFORBODY_END;
@@ -788,7 +786,7 @@ public class ProgramConverter
sb.append( PARFORBODY_END );
- return sb.toString();
+ return sb.toString();
}
private static String serializeProgram( Program prog, ArrayList<ProgramBlock> pbs, HashMap<String, byte[]> clsMap )
@@ -797,7 +795,7 @@ public class ProgramConverter
//note program contains variables, programblocks and function program blocks
//but in order to avoid redundancy, we only serialize function program blocks
- HashMap<String, FunctionProgramBlock> fpb = prog.getFunctionProgramBlocks();
+ HashMap<String, FunctionProgramBlock> fpb = prog.getFunctionProgramBlocks();
HashSet<String> cand = new HashSet<String>();
rFindSerializationCandidates(pbs, cand);
@@ -812,7 +810,7 @@ public class ProgramConverter
if( pb instanceof WhileProgramBlock )
{
WhileProgramBlock wpb = (WhileProgramBlock) pb;
- rFindSerializationCandidates(wpb.getChildBlocks(), cand );
+ rFindSerializationCandidates(wpb.getChildBlocks(), cand );
}
else if ( pb instanceof ForProgramBlock || pb instanceof ParForProgramBlock )
{
@@ -870,7 +868,7 @@ public class ProgramConverter
DataType datatype = dat.getDataType();
ValueType valuetype = dat.getValueType();
String value = null;
- String[] matrixMetaData = null;
+ String[] matrixMetaData = null;
switch( datatype )
{
@@ -1041,24 +1039,6 @@ public class ProgramConverter
return sb.toString();
}
- private static String serializeStringArray( String[] vars)
- {
- StringBuilder sb = new StringBuilder();
- int count=0;
- for( String s : vars )
- {
- if(count>0)
- sb.append( ELEMENT_DELIM );
- if( s != null )
- sb.append( s );
- else
- sb.append( "null" );
-
- count++;
- }
- return sb.toString();
- }
-
private static String serializeDataIdentifiers( ArrayList<DataIdentifier> var)
{
StringBuilder sb = new StringBuilder();
@@ -1157,8 +1137,6 @@ public class ProgramConverter
sb.append( serializeInstructions( wpb.getPredicate(), clsMap ) );
sb.append( PARFOR_INST_END );
sb.append( COMPONENTS_DELIM );
- sb.append( wpb.getPredicateResultVar() );
- sb.append( COMPONENTS_DELIM );
sb.append( PARFOR_INST_BEGIN );
sb.append( serializeInstructions( wpb.getExitInstructions(), clsMap ) );
sb.append( PARFOR_INST_END );
@@ -1170,7 +1148,7 @@ public class ProgramConverter
else if ( pb instanceof ForProgramBlock && !(pb instanceof ParForProgramBlock ) )
{
ForProgramBlock fpb = (ForProgramBlock) pb;
- sb.append( serializeStringArray(fpb.getIterablePredicateVars()) );
+ sb.append( fpb.getIterVar() );
sb.append( COMPONENTS_DELIM );
sb.append( PARFOR_INST_BEGIN );
sb.append( serializeInstructions( fpb.getFromInstructions(), clsMap ) );
@@ -1200,7 +1178,7 @@ public class ProgramConverter
if( PExecMode.valueOf( pfpb.getParForParams().get( ParForStatementBlock.EXEC_MODE )) == PExecMode.REMOTE_MR )
throw new DMLRuntimeException( NOT_SUPPORTED_MR_PARFOR );
- sb.append( serializeStringArray(pfpb.getIterablePredicateVars()) );
+ sb.append( pfpb.getIterVar() );
sb.append( COMPONENTS_DELIM );
sb.append( serializeStringArrayList( pfpb.getResultVariables()) );
sb.append( COMPONENTS_DELIM );
@@ -1231,9 +1209,7 @@ public class ProgramConverter
IfProgramBlock ipb = (IfProgramBlock) pb;
sb.append( PARFOR_INST_BEGIN );
sb.append( serializeInstructions(ipb.getPredicate(), clsMap) );
- sb.append( PARFOR_INST_END );
- sb.append( COMPONENTS_DELIM );
- sb.append( ipb.getPredicateResultVar() );
+ sb.append( PARFOR_INST_END );
sb.append( COMPONENTS_DELIM );
sb.append( PARFOR_INST_BEGIN );
sb.append( serializeInstructions(ipb.getExitInstructions(), clsMap) );
@@ -1470,11 +1446,8 @@ public class ProgramConverter
String lin = in.substring( PARFOR_PB_WHILE.length(),in.length()-PARFOR_PB_END.length());
HierarchyAwareStringTokenizer st = new HierarchyAwareStringTokenizer(lin, COMPONENTS_DELIM);
- //LocalVariableMap vars = parseVariables(st.nextToken());
-
//predicate instructions
ArrayList<Instruction> inst = parseInstructions(st.nextToken(),id);
- String var = st.nextToken();
//exit instructions
ArrayList<Instruction> exit = parseInstructions(st.nextToken(),id);
@@ -1483,10 +1456,8 @@ public class ProgramConverter
ArrayList<ProgramBlock> pbs = rParseProgramBlocks(st.nextToken(), prog, id);
WhileProgramBlock wpb = new WhileProgramBlock(prog,inst);
- wpb.setPredicateResultVar(var);
wpb.setExitInstructions2(exit);
wpb.setChildBlocks(pbs);
- //wpb.setVariables(vars);
return wpb;
}
@@ -1497,10 +1468,8 @@ public class ProgramConverter
String lin = in.substring( PARFOR_PB_FOR.length(),in.length()-PARFOR_PB_END.length());
HierarchyAwareStringTokenizer st = new HierarchyAwareStringTokenizer(lin, COMPONENTS_DELIM);
- //LocalVariableMap vars = parseVariables(st.nextToken());
-
//inputs
- String[] iterPredVars = parseStringArray(st.nextToken());
+ String iterVar = st.nextToken();
//instructions
ArrayList<Instruction> from = parseInstructions(st.nextToken(),id);
@@ -1509,17 +1478,16 @@ public class ProgramConverter
//exit instructions
ArrayList<Instruction> exit = parseInstructions(st.nextToken(),id);
-
+
//program blocks
ArrayList<ProgramBlock> pbs = rParseProgramBlocks(st.nextToken(), prog, id);
-
- ForProgramBlock fpb = new ForProgramBlock(prog, iterPredVars);
+
+ ForProgramBlock fpb = new ForProgramBlock(prog, iterVar);
fpb.setFromInstructions(from);
fpb.setToInstructions(to);
fpb.setIncrementInstructions(incr);
fpb.setExitInstructions(exit);
fpb.setChildBlocks(pbs);
- //fpb.setVariables(vars);
return fpb;
}
@@ -1530,10 +1498,8 @@ public class ProgramConverter
String lin = in.substring( PARFOR_PB_PARFOR.length(),in.length()-PARFOR_PB_END.length());
HierarchyAwareStringTokenizer st = new HierarchyAwareStringTokenizer(lin, COMPONENTS_DELIM);
- //LocalVariableMap vars = parseVariables(st.nextToken());
-
//inputs
- String[] iterPredVars = parseStringArray(st.nextToken());
+ String iterVar = st.nextToken();
ArrayList<String> resultVars = parseStringArrayList(st.nextToken());
HashMap<String,String> params = parseStringHashMap(st.nextToken());
@@ -1544,11 +1510,11 @@ public class ProgramConverter
//exit instructions
ArrayList<Instruction> exit = parseInstructions(st.nextToken(), 0);
-
+
//program blocks //reset id to preinit state, replaced during exec
ArrayList<ProgramBlock> pbs = rParseProgramBlocks(st.nextToken(), prog, 0);
-
- ParForProgramBlock pfpb = new ParForProgramBlock(id, prog, iterPredVars, params);
+
+ ParForProgramBlock pfpb = new ParForProgramBlock(id, prog, iterVar, params);
pfpb.disableOptimization(); //already done in top-level parfor
pfpb.setResultVariables(resultVars);
pfpb.setFromInstructions(from);
@@ -1556,22 +1522,18 @@ public class ProgramConverter
pfpb.setIncrementInstructions(incr);
pfpb.setExitInstructions(exit);
pfpb.setChildBlocks(pbs);
- //pfpb.setVariables(vars);
return pfpb;
}
- private static IfProgramBlock rParseIfProgramBlock( String in, Program prog, int id )
+ private static IfProgramBlock rParseIfProgramBlock( String in, Program prog, int id )
throws DMLRuntimeException
{
- String lin = in.substring( PARFOR_PB_IF.length(),in.length()-PARFOR_PB_END.length());
+ String lin = in.substring( PARFOR_PB_IF.length(),in.length()-PARFOR_PB_END.length());
HierarchyAwareStringTokenizer st = new HierarchyAwareStringTokenizer(lin, COMPONENTS_DELIM);
- //LocalVariableMap vars = parseVariables(st.nextToken());
-
//predicate instructions
ArrayList<Instruction> inst = parseInstructions(st.nextToken(),id);
- String var = st.nextToken();
//exit instructions
ArrayList<Instruction> exit = parseInstructions(st.nextToken(),id);
@@ -1581,23 +1543,19 @@ public class ProgramConverter
ArrayList<ProgramBlock> pbs2 = rParseProgramBlocks(st.nextToken(), prog, id);
IfProgramBlock ipb = new IfProgramBlock(prog,inst);
- ipb.setPredicateResultVar(var);
ipb.setExitInstructions2(exit);
ipb.setChildBlocksIfBody(pbs1);
ipb.setChildBlocksElseBody(pbs2);
- //ipb.setVariables(vars);
return ipb;
}
- private static FunctionProgramBlock rParseFunctionProgramBlock( String in, Program prog, int id )
+ private static FunctionProgramBlock rParseFunctionProgramBlock( String in, Program prog, int id )
throws DMLRuntimeException
{
- String lin = in.substring( PARFOR_PB_FC.length(),in.length()-PARFOR_PB_END.length());
+ String lin = in.substring( PARFOR_PB_FC.length(),in.length()-PARFOR_PB_END.length());
HierarchyAwareStringTokenizer st = new HierarchyAwareStringTokenizer(lin, COMPONENTS_DELIM);
- //LocalVariableMap vars = parseVariables(st.nextToken());
-
//inputs and outputs
ArrayList<DataIdentifier> dat1 = parseDataIdentifiers(st.nextToken());
ArrayList<DataIdentifier> dat2 = parseDataIdentifiers(st.nextToken());
@@ -1613,7 +1571,6 @@ public class ProgramConverter
FunctionProgramBlock fpb = new FunctionProgramBlock(prog, tmp1, tmp2);
fpb.setInstructions(inst);
fpb.setChildBlocks(pbs);
- //fpb.setVariables(vars);
return fpb;
}
@@ -1645,9 +1602,7 @@ public class ProgramConverter
//only CP external functions, because no nested MR jobs for reblocks
ExternalFunctionProgramBlockCP efpb = new ExternalFunctionProgramBlockCP(prog, tmp1, tmp2, dat3, basedir);
- //efpb.setInstructions(inst);
efpb.setChildBlocks(pbs);
- //efpb.setVariables(vars);
return efpb;
}
@@ -1713,37 +1668,19 @@ public class ProgramConverter
{
ArrayList<String> vars = new ArrayList<String>();
StringTokenizer st = new StringTokenizer(in,ELEMENT_DELIM);
- while( st.hasMoreTokens() )
- {
- String tmp = st.nextToken();
+ while( st.hasMoreTokens() ) {
+ String tmp = st.nextToken();
vars.add(tmp);
}
return vars;
}
- private static String[] parseStringArray( String in )
- {
- StringTokenizer st = new StringTokenizer(in, ELEMENT_DELIM);
- int len = st.countTokens();
- String[] a = new String[len];
- for( int i=0; i<len; i++ )
- {
- String tmp = st.nextToken();
- if( tmp.equals("null") )
- a[i] = null;
- else
- a[i] = tmp;
- }
- return a;
- }
-
private static ArrayList<DataIdentifier> parseDataIdentifiers( String in )
{
ArrayList<DataIdentifier> vars = new ArrayList<DataIdentifier>();
StringTokenizer st = new StringTokenizer(in, ELEMENT_DELIM);
- while( st.hasMoreTokens() )
- {
+ while( st.hasMoreTokens() ) {
String tmp = st.nextToken();
DataIdentifier dat = parseDataIdentifier( tmp );
vars.add(dat);
@@ -1807,7 +1744,7 @@ public class ProgramConverter
dat = new StringObject(name,valString);
break;
default:
- throw new DMLRuntimeException("Unable to parse valuetype "+valuetype);
+ throw new DMLRuntimeException("Unable to parse valuetype "+valuetype);
}
break;
}
@@ -1820,7 +1757,7 @@ public class ProgramConverter
int bcols = Integer.parseInt( st.nextToken() );
long nnz = Long.parseLong( st.nextToken() );
InputInfo iin = InputInfo.stringToInputInfo( st.nextToken() );
- OutputInfo oin = OutputInfo.stringToOutputInfo( st.nextToken() );
+ OutputInfo oin = OutputInfo.stringToOutputInfo( st.nextToken() );
PartitionFormat partFormat = PartitionFormat.valueOf( st.nextToken() );
UpdateType inplace = UpdateType.valueOf( st.nextToken() );
MatrixCharacteristics mc = new MatrixCharacteristics(rows, cols, brows, bcols, nnz);
@@ -1940,7 +1877,7 @@ public class ProgramConverter
public String nextToken()
{
- int nextDelim = determineNextSameLevelIndexOf(_str, _del);
+ int nextDelim = determineNextSameLevelIndexOf(_str, _del);
String token = null;
if(nextDelim < 0)
{
@@ -1965,7 +1902,7 @@ public class ProgramConverter
i2 = tmpdata.indexOf(LEVELIN);
i3 = tmpdata.indexOf(LEVELOUT);
- if( i1 < 0 ) return i1; //no pattern found at all
+ if( i1 < 0 ) return i1; //no pattern found at all
min = i1; //min >= 0 by definition
if( i2 >= 0 ) min = Math.min(min, i2);
http://git-wip-us.apache.org/repos/asf/systemml/blob/1d83cedb/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptTreeConverter.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptTreeConverter.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptTreeConverter.java
index 1d5a195..d8cad78 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptTreeConverter.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptTreeConverter.java
@@ -181,8 +181,9 @@ public class OptTreeConverter
_rtMap.putMapping(fpb, node);
node.setExecType(ExecType.CP);
- //TODO use constant value if known
- node.addParam(ParamType.NUM_ITERATIONS, String.valueOf(CostEstimator.FACTOR_NUM_ITERATIONS));
+ //determine number of iterations
+ long N = OptimizerUtils.getNumIterations(fpb, vars, CostEstimator.FACTOR_NUM_ITERATIONS);
+ node.addParam(ParamType.NUM_ITERATIONS, String.valueOf(N));
node.addChilds( createOptNodes( fpb.getFromInstructions(), vars,storeObjs ) );
node.addChilds( createOptNodes( fpb.getToInstructions(), vars,storeObjs ) );
@@ -375,7 +376,9 @@ public class OptTreeConverter
node.setExecType(ExecType.CP);
node.setLineNumbers(fsb.getBeginLine(), fsb.getEndLine());
- node.addParam(ParamType.NUM_ITERATIONS, String.valueOf(CostEstimator.FACTOR_NUM_ITERATIONS));
+ //determine number of iterations
+ long N = OptimizerUtils.getNumIterations(fpb, vars, CostEstimator.FACTOR_NUM_ITERATIONS);
+ node.addParam(ParamType.NUM_ITERATIONS, String.valueOf(N));
//handle predicate
fsb.getFromHops().resetVisitStatus();
http://git-wip-us.apache.org/repos/asf/systemml/blob/1d83cedb/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerConstrained.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerConstrained.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerConstrained.java
index fb83fd6..9e78c2a 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerConstrained.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerConstrained.java
@@ -154,14 +154,11 @@ public class OptimizerConstrained extends OptimizerRuleBased
// rewrite 8: rewrite set partition replication factor
super.rewriteSetExportReplicationFactor( pn, ec.getVariables() );
- // rewrite 9: nested parallelism (incl exec types)
- boolean flagNested = super.rewriteNestedParallelism( pn, M1, flagLIX );
-
// rewrite 10: determine parallelism
- rewriteSetDegreeOfParallelism( pn, M1, flagNested );
+ rewriteSetDegreeOfParallelism( pn, M1, false );
// rewrite 11: task partitioning
- rewriteSetTaskPartitioner( pn, flagNested, flagLIX );
+ rewriteSetTaskPartitioner( pn, false, flagLIX );
// rewrite 12: fused data partitioning and execution
rewriteSetFusedDataPartitioningExecution(pn, M1, flagLIX, partitionedMatrices, ec.getVariables(), tmpmode);
@@ -369,11 +366,10 @@ public class OptimizerConstrained extends OptimizerRuleBased
if( emode == PExecMode.REMOTE_MR_DP || emode == PExecMode.REMOTE_SPARK_DP )
{
ParForProgramBlock pfpb = (ParForProgramBlock) OptTreeConverter
- .getAbstractPlanMapping().getMappedProg(pn.getID())[1];
+ .getAbstractPlanMapping().getMappedProg(pn.getID())[1];
//partitioned matrix
- if( partitionedMatrices.size()<=0 )
- {
+ if( partitionedMatrices.size()<=0 ) {
LOG.debug(getOptMode()+" OPT: unable to force 'set fused data partitioning and execution' - result="+false );
return;
}
@@ -381,11 +377,8 @@ public class OptimizerConstrained extends OptimizerRuleBased
String moVarname = partitionedMatrices.keySet().iterator().next();
PartitionFormat moDpf = partitionedMatrices.get(moVarname);
MatrixObject mo = (MatrixObject)vars.get(moVarname);
-
- //check if access via iteration variable and sizes match
- String iterVarname = pfpb.getIterablePredicateVars()[0];
-
- if( rIsAccessByIterationVariable(pn, moVarname, iterVarname) &&
+
+ if( rIsAccessByIterationVariable(pn, moVarname, pfpb.getIterVar()) &&
((moDpf==PartitionFormat.ROW_WISE && mo.getNumRows()==_N ) ||
(moDpf==PartitionFormat.COLUMN_WISE && mo.getNumColumns()==_N) ||
(moDpf._dpf==PDataPartitionFormat.ROW_BLOCK_WISE_N && mo.getNumRows()<=_N*moDpf._N)||
http://git-wip-us.apache.org/repos/asf/systemml/blob/1d83cedb/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java
index 118aede..018364f 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java
@@ -155,7 +155,6 @@ public class OptimizerRuleBased extends Optimizer
public static final int MAX_REPLICATION_FACTOR_PARTITIONING = 5;
public static final int MAX_REPLICATION_FACTOR_EXPORT = 7;
public static final boolean ALLOW_REMOTE_NESTED_PARALLELISM = false;
- public static final boolean APPLY_REWRITE_NESTED_PARALLELISM = false;
public static final String FUNCTION_UNFOLD_NAMEPREFIX = "__unfold_";
public static final boolean APPLY_REWRITE_UPDATE_INPLACE_INTERMEDIATE = true;
@@ -285,14 +284,11 @@ public class OptimizerRuleBased extends Optimizer
// rewrite 8: rewrite set partition replication factor
rewriteSetExportReplicationFactor( pn, ec.getVariables() );
- // rewrite 9: nested parallelism (incl exec types)
- boolean flagNested = rewriteNestedParallelism( pn, M1, flagLIX );
-
// rewrite 10: determine parallelism
- rewriteSetDegreeOfParallelism( pn, M1, flagNested );
+ rewriteSetDegreeOfParallelism( pn, M1, false );
// rewrite 11: task partitioning
- rewriteSetTaskPartitioner( pn, flagNested, flagLIX );
+ rewriteSetTaskPartitioner( pn, false, flagLIX );
// rewrite 12: fused data partitioning and execution
rewriteSetFusedDataPartitioningExecution(pn, M1, flagLIX, partitionedMatrices, ec.getVariables());
@@ -357,7 +353,7 @@ public class OptimizerRuleBased extends Optimizer
protected void analyzeProblemAndInfrastructure( OptNode pn )
{
- _N = Long.parseLong(pn.getParam(ParamType.NUM_ITERATIONS));
+ _N = Long.parseLong(pn.getParam(ParamType.NUM_ITERATIONS));
_Nmax = pn.getMaxProblemSize();
_lk = InfrastructureAnalyzer.getLocalParallelism();
_lkmaxCP = (int) Math.ceil( PAR_K_FACTOR * _lk );
@@ -617,9 +613,9 @@ public class OptimizerRuleBased extends Optimizer
//determine if applicable
boolean apply = M < _rm //ops fit in remote memory budget
&& !cand.isEmpty() //at least one MR
- && isResultPartitionableAll(cand,pfpb.getResultVariables(),
- vars, pfpb.getIterablePredicateVars()[0]); // check candidates
-
+ && isResultPartitionableAll(cand,pfpb.getResultVariables(),
+ vars, pfpb.getIterVar()); // check candidates
+
//recompile LIX
if( apply )
{
@@ -1002,7 +998,7 @@ public class OptimizerRuleBased extends Optimizer
{
//find all candidates matrices (at least one partitioned access via iterVar)
HashSet<String> cand = new HashSet<String>();
- rFindDataColocationCandidates(n, cand, pfpb.getIterablePredicateVars()[0]);
+ rFindDataColocationCandidates(n, cand, pfpb.getIterVar());
//select largest matrix for colocation (based on nnz to account for sparsity)
long nnzMax = Long.MIN_VALUE;
@@ -1016,7 +1012,7 @@ public class OptimizerRuleBased extends Optimizer
apply = true;
}
}
- }
+ }
}
//modify the runtime plan (apply true if at least one candidate)
@@ -1093,7 +1089,7 @@ public class OptimizerRuleBased extends Optimizer
if(((n.getExecType()==ExecType.MR && n.getParam(ParamType.DATA_PARTITIONER).equals(PDataPartitioner.REMOTE_MR.name()))
|| (n.getExecType()==ExecType.SPARK && n.getParam(ParamType.DATA_PARTITIONER).equals(PDataPartitioner.REMOTE_SPARK.name())))
&& n.hasNestedParallelism(false)
- && n.hasNestedPartitionReads(false) )
+ && n.hasNestedPartitionReads(false) )
{
apply = true;
@@ -1179,73 +1175,6 @@ public class OptimizerRuleBased extends Optimizer
_numEvaluatedPlans++;
LOG.debug(getOptMode()+" OPT: rewrite 'set export replication factor' - result="+apply+((apply)?" ("+replication+")":"") );
}
-
-
- ///////
- //REWRITE enable nested parallelism
- ///
-
- @SuppressWarnings("all")
- protected boolean rewriteNestedParallelism(OptNode n, double M, boolean flagLIX )
- throws DMLRuntimeException
- {
- boolean nested = false;
-
- if( APPLY_REWRITE_NESTED_PARALLELISM
- && !flagLIX // if not applied left indexing rewrite
- && _N >= _rnk // at least exploit all nodes
- && !n.hasNestedParallelism(false)// only for 1D problems, otherwise potentially bad load balance
- && M * _lkmaxCP <= _rm ) // only if we can exploit full local parallelism in the map task JVM memory
- {
- //modify tree
- ArrayList<OptNode> tmpOld = n.getChilds();
- OptNode nest = new OptNode(NodeType.PARFOR, ExecType.CP);
- ArrayList<OptNode> tmpNew = new ArrayList<OptNode>();
- tmpNew.add(nest);
- n.setChilds(tmpNew);
- nest.setChilds(tmpOld);
-
- //modify rtprog
- long id = n.getID();
- ParForProgramBlock pfpb = (ParForProgramBlock) OptTreeConverter
- .getAbstractPlanMapping().getMappedProg(id)[1];
- ArrayList<ProgramBlock> tmpPBOld = pfpb.getChildBlocks();
-
- //create new program block structure and modify parameters (from, to, incr, types,)
- String[] iterVars = pfpb.getIterablePredicateVars(); //from, to stay original
- String[] iterVars2 = iterVars.clone(); //itervar, incr stay original
- int outIncr = (int)Math.ceil(((double)_N)/_rnk);
- iterVars[ 0 ] = ParForStatementBlock.INTERAL_FN_INDEX_ROW; // already checked for uniqueness in ParForStatementBlock
- iterVars[ 3 ] = String.valueOf(outIncr);
- iterVars2[ 1 ] = ParForStatementBlock.INTERAL_FN_INDEX_ROW; //sub start
- iterVars2[ 2 ] = null;
- HashMap<String,String> params = pfpb.getParForParams();
- HashMap<String,String> params2 = (HashMap<String,String>)params.clone();
- ParForProgramBlock pfpb2 = new ParForProgramBlock(pfpb.getProgram(),iterVars2, params2);
- OptTreeConverter.getAbstractPlanMapping().putProgMapping(null, pfpb2, nest);
-
- ArrayList<ProgramBlock> tmpPBNew = new ArrayList<ProgramBlock>();
- tmpPBNew.add(pfpb2);
- pfpb.setChildBlocks(tmpPBNew);
- pfpb.setIterablePredicateVars(iterVars);
- pfpb.setIncrementInstructions(new ArrayList<Instruction>());
- pfpb.setExecMode(PExecMode.REMOTE_MR);
- pfpb2.setChildBlocks(tmpPBOld);
- pfpb2.setResultVariables(pfpb.getResultVariables());
- pfpb2.setFromInstructions(new ArrayList<Instruction>());
- pfpb2.setToInstructions(ProgramRecompiler.createNestedParallelismToInstructionSet( ParForStatementBlock.INTERAL_FN_INDEX_ROW, String.valueOf(outIncr-1) ));
- pfpb2.setIncrementInstructions(new ArrayList<Instruction>());
- pfpb2.setExecMode(PExecMode.LOCAL);
-
- nested = true;
- }
-
- _numEvaluatedPlans++;
- LOG.debug(getOptMode()+" OPT: rewrite 'enable nested parallelism' - result="+nested );
-
- return nested;
- }
-
///////
//REWRITE set degree of parallelism
@@ -1526,10 +1455,7 @@ public class OptimizerRuleBased extends Optimizer
PartitionFormat moDpf = partitionedMatrices.get(moVarname);
MatrixObject mo = (MatrixObject)vars.get(moVarname);
- //check if access via iteration variable and sizes match
- String iterVarname = pfpb.getIterablePredicateVars()[0];
-
- if( rIsAccessByIterationVariable(pn, moVarname, iterVarname) &&
+ if( rIsAccessByIterationVariable(pn, moVarname, pfpb.getIterVar()) &&
((moDpf==PartitionFormat.ROW_WISE && mo.getNumRows()==_N ) ||
(moDpf==PartitionFormat.COLUMN_WISE && mo.getNumColumns()==_N) ||
(moDpf._dpf==PDataPartitionFormat.ROW_BLOCK_WISE_N && mo.getNumRows()<=_N*moDpf._N)||
@@ -1540,7 +1466,7 @@ public class OptimizerRuleBased extends Optimizer
pn.addParam(ParamType.DATA_PARTITIONER, REMOTE_DPE.toString()+"(fused)");
pn.setK( k );
- pfpb.setExecMode(REMOTE_DPE); //set fused exec type
+ pfpb.setExecMode(REMOTE_DPE); //set fused exec type
pfpb.setDataPartitioner(PDataPartitioner.NONE);
pfpb.enableColocatedPartitionedMatrix( moVarname );
pfpb.setDegreeOfParallelism(k);
@@ -2815,7 +2741,7 @@ public class OptimizerRuleBased extends Optimizer
ArrayList<String> cleanedVars = new ArrayList<String>();
ArrayList<String> resultVars = pfpb.getResultVariables();
- String itervar = pfpb.getIterablePredicateVars()[0];
+ String itervar = pfpb.getIterVar();
for( String rvar : resultVars ) {
Data dat = ec.getVariable(rvar);
http://git-wip-us.apache.org/repos/asf/systemml/blob/1d83cedb/src/test/java/org/apache/sysml/test/gpu/RightIndexingTests.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/gpu/RightIndexingTests.java b/src/test/java/org/apache/sysml/test/gpu/RightIndexingTests.java
index e891b7f..90b08ae 100644
--- a/src/test/java/org/apache/sysml/test/gpu/RightIndexingTests.java
+++ b/src/test/java/org/apache/sysml/test/gpu/RightIndexingTests.java
@@ -57,7 +57,7 @@ public class RightIndexingTests extends GPUTests {
for (int k = 0; k < sparsities.length; k++) {
double sparsity = sparsities[k];
Matrix X = generateInputMatrix(spark, dim1, dim2, sparsity, seed);
- Matrix Y = generateInputMatrix(spark, dim1, dim2, sparsity, seed);
+ //FIXME Matrix Y = generateInputMatrix(spark, dim1, dim2, sparsity, seed);
HashMap<String, Object> inputs = new HashMap<>();
inputs.put("X", X);
String scriptStr = "O = X[" + rl + ":" + ru + "," + cl + ":" + cu + "];";
http://git-wip-us.apache.org/repos/asf/systemml/blob/1d83cedb/src/test/java/org/apache/sysml/test/integration/functions/recompile/SparsityRecompileTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/recompile/SparsityRecompileTest.java b/src/test/java/org/apache/sysml/test/integration/functions/recompile/SparsityRecompileTest.java
index 923dc47..c8a883a 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/recompile/SparsityRecompileTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/recompile/SparsityRecompileTest.java
@@ -47,8 +47,8 @@ public class SparsityRecompileTest extends AutomatedTestBase
private final static String TEST_CLASS_DIR = TEST_DIR + SparsityRecompileTest.class.getSimpleName() + "/";
private final static long rows = 1000;
- private final static long cols = 500000;
- private final static double sparsity = 0.00001d;
+ private final static long cols = 500000;
+ private final static double sparsity = 0.00001d;
private final static double val = 7.0;
@Override
http://git-wip-us.apache.org/repos/asf/systemml/blob/1d83cedb/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/SVDFactorizeTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/SVDFactorizeTest.java b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/SVDFactorizeTest.java
index 4f09b74..8623d32 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/SVDFactorizeTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/SVDFactorizeTest.java
@@ -29,7 +29,6 @@ import org.apache.sysml.test.integration.TestConfiguration;
public class SVDFactorizeTest extends AutomatedTestBase
{
-
private final static String TEST_NAME1 = "svd";
private final static String TEST_DIR = "functions/unary/matrix/";
private static final String TEST_CLASS_DIR = TEST_DIR + SVDFactorizeTest.class.getSimpleName() + "/";
@@ -41,59 +40,48 @@ public class SVDFactorizeTest extends AutomatedTestBase
private final static double sparsity = 0.9;
@Override
- public void setUp()
- {
- addTestConfiguration(
- TEST_NAME1,
- new TestConfiguration(TEST_CLASS_DIR, TEST_NAME1,
- new String[] { "D" }) );
+ public void setUp() {
+ addTestConfiguration(TEST_NAME1,
+ new TestConfiguration(TEST_CLASS_DIR, TEST_NAME1, new String[] { "D" }) );
}
@Test
- public void testSVDFactorizeDenseCP()
- {
+ public void testSVDFactorizeDenseCP() {
runTestSVDFactorize( rows1, cols1, RUNTIME_PLATFORM.SINGLE_NODE );
}
@Test
- public void testSVDFactorizeDenseSP()
- {
+ public void testSVDFactorizeDenseSP() {
runTestSVDFactorize( rows1, cols1, RUNTIME_PLATFORM.SPARK );
}
@Test
- public void testSVDFactorizeDenseMR()
- {
+ public void testSVDFactorizeDenseMR() {
runTestSVDFactorize( rows1, cols1, RUNTIME_PLATFORM.HADOOP );
}
@Test
- public void testSVDFactorizeDenseHybrid()
- {
+ public void testSVDFactorizeDenseHybrid() {
runTestSVDFactorize( rows1, cols1, RUNTIME_PLATFORM.HYBRID );
}
@Test
- public void testLargeSVDFactorizeDenseCP()
- {
+ public void testLargeSVDFactorizeDenseCP() {
runTestSVDFactorize( rows2, cols2, RUNTIME_PLATFORM.SINGLE_NODE );
}
@Test
- public void testLargeSVDFactorizeDenseSP()
- {
+ public void testLargeSVDFactorizeDenseSP() {
runTestSVDFactorize( rows2, cols2, RUNTIME_PLATFORM.SPARK );
}
@Test
- public void testLargeSVDFactorizeDenseMR()
- {
+ public void testLargeSVDFactorizeDenseMR() {
runTestSVDFactorize( rows2, cols2, RUNTIME_PLATFORM.HADOOP );
}
@Test
- public void testLargeSVDFactorizeDenseHybrid()
- {
+ public void testLargeSVDFactorizeDenseHybrid() {
runTestSVDFactorize( rows2, cols2, RUNTIME_PLATFORM.HYBRID );
}
@@ -103,8 +91,8 @@ public class SVDFactorizeTest extends AutomatedTestBase
rtplatform = rt;
boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
- if( rtplatform == RUNTIME_PLATFORM.SPARK )
- DMLScript.USE_LOCAL_SPARK_CONFIG = true;
+ if( rtplatform == RUNTIME_PLATFORM.SPARK )
+ DMLScript.USE_LOCAL_SPARK_CONFIG = true;
try
{
@@ -127,9 +115,10 @@ public class SVDFactorizeTest extends AutomatedTestBase
runTest(true, exceptionExpected, null, -1);
compareResults(1e-8);
}
- finally
- {
+ finally {
+ DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
rtplatform = rtold;
+
}
}
[2/2] systemml git commit: [SYSTEMML-1871] Rework compiler/runtime
predicate handling (if, loops)
Posted by mb...@apache.org.
[SYSTEMML-1871] Rework compiler/runtime predicate handling (if, loops)
So far we handled literal and expression predicates differently in
blocks such as if, while, for, and parfor. This was because DAGs of data
ops and literals do not compile to any instructions. The two different
code paths unnecessarily complicated the compiler and runtime,
especially with regard to rewrites such as constant folding, which can
cause a transition between the two code paths. We now properly compile a
transient write HOP on top of any predicate DAGs, which unifies two
cases and simplifies both compiler and runtime.
Furthermore, this patch also includes improvements for determining the
number of iterations during parfor optimization for loops inside the
surrounding parfor body, as well as a couple of minor cleanups.
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/1d83cedb
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/1d83cedb
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/1d83cedb
Branch: refs/heads/master
Commit: 1d83cedb7c569d8fd9571e6f76d502e1e622ed02
Parents: 100f2d6
Author: Matthias Boehm <mb...@gmail.com>
Authored: Thu Aug 24 22:16:19 2017 -0700
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Thu Aug 24 22:16:19 2017 -0700
----------------------------------------------------------------------
.../org/apache/sysml/api/jmlc/JMLCUtils.java | 2 +-
src/main/java/org/apache/sysml/hops/Hop.java | 2 +-
.../org/apache/sysml/hops/OptimizerUtils.java | 36 ++++-
.../apache/sysml/hops/cost/CostEstimator.java | 27 +---
.../apache/sysml/hops/recompile/Recompiler.java | 67 ++++------
.../hops/rewrite/RewriteConstantFolding.java | 17 +--
.../RewriteRemoveUnnecessaryBranches.java | 2 +-
.../sysml/parser/ConditionalPredicate.java | 6 +-
.../org/apache/sysml/parser/DMLTranslator.java | 130 ++++++-------------
.../org/apache/sysml/parser/IfStatement.java | 1 -
.../apache/sysml/parser/IfStatementBlock.java | 1 -
.../apache/sysml/parser/IterablePredicate.java | 118 +++++++----------
.../runtime/controlprogram/ForProgramBlock.java | 99 ++++++--------
.../runtime/controlprogram/IfProgramBlock.java | 91 ++-----------
.../controlprogram/ParForProgramBlock.java | 106 +++++----------
.../runtime/controlprogram/ProgramBlock.java | 56 +++-----
.../controlprogram/WhileProgramBlock.java | 87 ++-----------
.../controlprogram/parfor/ProgramConverter.java | 129 +++++-------------
.../parfor/opt/OptTreeConverter.java | 9 +-
.../parfor/opt/OptimizerConstrained.java | 19 +--
.../parfor/opt/OptimizerRuleBased.java | 98 ++------------
.../sysml/test/gpu/RightIndexingTests.java | 2 +-
.../recompile/SparsityRecompileTest.java | 4 +-
.../unary/matrix/SVDFactorizeTest.java | 43 +++---
24 files changed, 353 insertions(+), 799 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/1d83cedb/src/main/java/org/apache/sysml/api/jmlc/JMLCUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/jmlc/JMLCUtils.java b/src/main/java/org/apache/sysml/api/jmlc/JMLCUtils.java
index 60ac9d8..c09be9b 100644
--- a/src/main/java/org/apache/sysml/api/jmlc/JMLCUtils.java
+++ b/src/main/java/org/apache/sysml/api/jmlc/JMLCUtils.java
@@ -109,7 +109,7 @@ public class JMLCUtils
* @param outputs registered output variables
* @return list of instructions
*/
- public static ArrayList<Instruction> cleanupRuntimeInstructions( ArrayList<Instruction> insts, String[] outputs ) {
+ public static ArrayList<Instruction> cleanupRuntimeInstructions( ArrayList<Instruction> insts, String... outputs ) {
return cleanupRuntimeInstructions(insts, new HashSet<String>(Arrays.asList(outputs)));
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/1d83cedb/src/main/java/org/apache/sysml/hops/Hop.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/Hop.java b/src/main/java/org/apache/sysml/hops/Hop.java
index b48e227..eeaa5f1 100644
--- a/src/main/java/org/apache/sysml/hops/Hop.java
+++ b/src/main/java/org/apache/sysml/hops/Hop.java
@@ -1558,7 +1558,7 @@ public abstract class Hop implements ParseInfo
setDim2( size );
}
- public long computeSizeInformation( Hop input )
+ public static long computeSizeInformation( Hop input )
{
long ret = -1;
http://git-wip-us.apache.org/repos/asf/systemml/blob/1d83cedb/src/main/java/org/apache/sysml/hops/OptimizerUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/OptimizerUtils.java b/src/main/java/org/apache/sysml/hops/OptimizerUtils.java
index 3b98a34..a4143e3 100644
--- a/src/main/java/org/apache/sysml/hops/OptimizerUtils.java
+++ b/src/main/java/org/apache/sysml/hops/OptimizerUtils.java
@@ -40,8 +40,10 @@ import org.apache.sysml.lops.Checkpoint;
import org.apache.sysml.lops.Lop;
import org.apache.sysml.lops.LopProperties.ExecType;
import org.apache.sysml.lops.compile.Dag;
+import org.apache.sysml.parser.ForStatementBlock;
import org.apache.sysml.parser.Expression.ValueType;
import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.controlprogram.ForProgramBlock;
import org.apache.sysml.runtime.controlprogram.LocalVariableMap;
import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
import org.apache.sysml.runtime.controlprogram.parfor.ProgramConverter;
@@ -1126,7 +1128,39 @@ public class OptimizerUtils
return String.format("%.0f", inB/(1024*1024) );
}
-
+ public static long getNumIterations(ForProgramBlock fpb, long defaultValue) {
+ if( fpb.getStatementBlock()==null )
+ return defaultValue;
+ ForStatementBlock fsb = (ForStatementBlock) fpb.getStatementBlock();
+ try {
+ HashMap<Long,Long> memo = new HashMap<Long,Long>();
+ long from = rEvalSimpleLongExpression(fsb.getFromHops().getInput().get(0), memo);
+ long to = rEvalSimpleLongExpression(fsb.getToHops().getInput().get(0), memo);
+ long increment = (fsb.getIncrementHops()==null) ? (from < to) ? 1 : -1 :
+ rEvalSimpleLongExpression(fsb.getIncrementHops().getInput().get(0), memo);
+ if( from != Long.MAX_VALUE && to != Long.MAX_VALUE && increment != Long.MAX_VALUE )
+ return (int)Math.ceil(((double)(to-from+1))/increment);
+ }
+ catch(Exception ex){}
+ return defaultValue;
+ }
+
+ public static long getNumIterations(ForProgramBlock fpb, LocalVariableMap vars, long defaultValue) {
+ if( fpb.getStatementBlock()==null )
+ return defaultValue;
+ ForStatementBlock fsb = (ForStatementBlock) fpb.getStatementBlock();
+ try {
+ HashMap<Long,Long> memo = new HashMap<Long,Long>();
+ long from = rEvalSimpleLongExpression(fsb.getFromHops().getInput().get(0), memo, vars);
+ long to = rEvalSimpleLongExpression(fsb.getToHops().getInput().get(0), memo, vars);
+ long increment = (fsb.getIncrementHops()==null) ? (from < to) ? 1 : -1 :
+ rEvalSimpleLongExpression(fsb.getIncrementHops().getInput().get(0), memo);
+ if( from != Long.MAX_VALUE && to != Long.MAX_VALUE && increment != Long.MAX_VALUE )
+ return (int)Math.ceil(((double)(to-from+1))/increment);
+ }
+ catch(Exception ex){}
+ return defaultValue;
+ }
/**
* Function to evaluate simple size expressions over literals and now/ncol.
http://git-wip-us.apache.org/repos/asf/systemml/blob/1d83cedb/src/main/java/org/apache/sysml/hops/cost/CostEstimator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/cost/CostEstimator.java b/src/main/java/org/apache/sysml/hops/cost/CostEstimator.java
index 955fadb..6eff99b 100644
--- a/src/main/java/org/apache/sysml/hops/cost/CostEstimator.java
+++ b/src/main/java/org/apache/sysml/hops/cost/CostEstimator.java
@@ -28,6 +28,7 @@ import java.util.StringTokenizer;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.sysml.conf.ConfigurationManager;
+import org.apache.sysml.hops.OptimizerUtils;
import org.apache.sysml.lops.Lop;
import org.apache.sysml.parser.DMLProgram;
import org.apache.sysml.runtime.DMLRuntimeException;
@@ -132,7 +133,7 @@ public abstract class CostEstimator
for( ProgramBlock pb2 : tmp.getChildBlocks() )
ret += rGetTimeEstimate(pb2, stats, memoFunc, recursive);
- ret *= getNumIterations(stats, tmp.getIterablePredicateVars());
+ ret *= getNumIterations(stats, tmp);
}
else if ( pb instanceof FunctionProgramBlock
&& !(pb instanceof ExternalFunctionProgramBlock)) //see generic
@@ -631,29 +632,15 @@ public abstract class CostEstimator
if( tmp!=null )
tmp._inmem = false; //all MR job outptus on HDFS
}
- }
+ }
- // TODO use of vars - needed for recompilation w/o exception
- private int getNumIterations(HashMap<String,VarStats> stats, String[] pred)
- {
- int N = DEFAULT_NUMITER;
- if( pred != null && pred[1]!=null && pred[2]!=null && pred[3]!=null )
- {
- try
- {
- int from = Integer.parseInt(pred[1]);
- int to = Integer.parseInt(pred[2]);
- int increment = Integer.parseInt(pred[3]);
- N = (int)Math.ceil(((double)(to-from+1))/increment);
- }
- catch(Exception ex){}
- }
- return N;
+ private long getNumIterations(HashMap<String,VarStats> stats, ForProgramBlock pb) {
+ return OptimizerUtils.getNumIterations(pb, DEFAULT_NUMITER);
}
- protected abstract double getCPInstTimeEstimate( Instruction inst, VarStats[] vs, String[] args )
+ protected abstract double getCPInstTimeEstimate( Instruction inst, VarStats[] vs, String[] args )
throws DMLRuntimeException;
- protected abstract double getMRJobInstTimeEstimate( Instruction inst, VarStats[] vs, String[] args )
+ protected abstract double getMRJobInstTimeEstimate( Instruction inst, VarStats[] vs, String[] args )
throws DMLRuntimeException;
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/1d83cedb/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java b/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java
index 091164d..f22d44a 100644
--- a/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java
+++ b/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java
@@ -1126,23 +1126,18 @@ public class Recompiler
private static void recompileIfPredicate( IfProgramBlock ipb, IfStatementBlock isb, LocalVariableMap vars, RecompileStatus status, long tid, boolean resetRecompile )
throws DMLRuntimeException, HopsException, LopsException, IOException
{
- if( isb != null )
- {
- Hop hops = isb.getPredicateHops();
- if( hops != null ) {
- ArrayList<Instruction> tmp = recompileHopsDag(
- hops, vars, status, true, false, tid);
- ipb.setPredicate( tmp );
- if( ParForProgramBlock.RESET_RECOMPILATION_FLAGs
- && resetRecompile )
- {
- Hop.resetRecompilationFlag(hops, ExecType.CP);
- isb.updatePredicateRecompilationFlag();
- }
-
- //update predicate vars (potentially after constant folding, e.g., in parfor)
- if( hops instanceof LiteralOp )
- ipb.setPredicateResultVar(((LiteralOp)hops).getName().toLowerCase());
+ if( isb == null )
+ return;
+
+ Hop hops = isb.getPredicateHops();
+ if( hops != null ) {
+ ArrayList<Instruction> tmp = recompileHopsDag(
+ hops, vars, status, true, false, tid);
+ ipb.setPredicate( tmp );
+ if( ParForProgramBlock.RESET_RECOMPILATION_FLAGs
+ && resetRecompile ) {
+ Hop.resetRecompilationFlag(hops, ExecType.CP);
+ isb.updatePredicateRecompilationFlag();
}
}
}
@@ -1150,23 +1145,18 @@ public class Recompiler
private static void recompileWhilePredicate( WhileProgramBlock wpb, WhileStatementBlock wsb, LocalVariableMap vars, RecompileStatus status, long tid, boolean resetRecompile )
throws DMLRuntimeException, HopsException, LopsException, IOException
{
- if( wsb != null )
- {
- Hop hops = wsb.getPredicateHops();
- if( hops != null ) {
- ArrayList<Instruction> tmp = recompileHopsDag(
- hops, vars, status, true, false, tid);
- wpb.setPredicate( tmp );
- if( ParForProgramBlock.RESET_RECOMPILATION_FLAGs
- && resetRecompile )
- {
- Hop.resetRecompilationFlag(hops, ExecType.CP);
- wsb.updatePredicateRecompilationFlag();
- }
-
- //update predicate vars (potentially after constant folding, e.g., in parfor)
- if( hops instanceof LiteralOp )
- wpb.setPredicateResultVar(((LiteralOp)hops).getName().toLowerCase());
+ if( wsb == null )
+ return;
+
+ Hop hops = wsb.getPredicateHops();
+ if( hops != null ) {
+ ArrayList<Instruction> tmp = recompileHopsDag(
+ hops, vars, status, true, false, tid);
+ wpb.setPredicate( tmp );
+ if( ParForProgramBlock.RESET_RECOMPILATION_FLAGs
+ && resetRecompile ) {
+ Hop.resetRecompilationFlag(hops, ExecType.CP);
+ wsb.updatePredicateRecompilationFlag();
}
}
}
@@ -1222,15 +1212,6 @@ public class Recompiler
fpb.setIncrementInstructions(tmp);
}
}
-
- //update predicate vars (potentially after constant folding, e.g., in parfor)
- String[] itervars = fpb.getIterablePredicateVars();
- if( fromHops != null && fromHops instanceof LiteralOp )
- itervars[1] = ((LiteralOp)fromHops).getName();
- if( toHops != null && toHops instanceof LiteralOp )
- itervars[2] = ((LiteralOp)toHops).getName();
- if( incrHops != null && incrHops instanceof LiteralOp )
- itervars[3] = ((LiteralOp)incrHops).getName();
}
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/1d83cedb/src/main/java/org/apache/sysml/hops/rewrite/RewriteConstantFolding.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/rewrite/RewriteConstantFolding.java b/src/main/java/org/apache/sysml/hops/rewrite/RewriteConstantFolding.java
index ac38138..ec9dcae 100644
--- a/src/main/java/org/apache/sysml/hops/rewrite/RewriteConstantFolding.java
+++ b/src/main/java/org/apache/sysml/hops/rewrite/RewriteConstantFolding.java
@@ -54,7 +54,6 @@ import org.apache.sysml.runtime.instructions.cp.ScalarObject;
*/
public class RewriteConstantFolding extends HopRewriteRule
{
-
private static final String TMP_VARNAME = "__cf_tmp";
//reuse basic execution runtime
@@ -69,8 +68,7 @@ public class RewriteConstantFolding extends HopRewriteRule
if( roots == null )
return null;
- for( int i=0; i<roots.size(); i++ )
- {
+ for( int i=0; i<roots.size(); i++ ) {
Hop h = roots.get(i);
roots.set(i, rule_ConstantFolding(h));
}
@@ -102,8 +100,7 @@ public class RewriteConstantFolding extends HopRewriteRule
//recursively process childs (before replacement to allow bottom-recursion)
//no iterator in order to prevent concurrent modification
- for( int i=0; i<root.getInput().size(); i++ )
- {
+ for( int i=0; i<root.getInput().size(); i++ ) {
Hop h = root.getInput().get(i);
rConstantFoldingExpression(h);
}
@@ -112,7 +109,7 @@ public class RewriteConstantFolding extends HopRewriteRule
//fold binary op if both are literals / unary op if literal
if( root.getDataType() == DataType.SCALAR //scalar ouput
- && ( isApplicableBinaryOp(root) || isApplicableUnaryOp(root) ) )
+ && ( isApplicableBinaryOp(root) || isApplicableUnaryOp(root) ) )
{
//core constant folding via runtime instructions
try {
@@ -134,7 +131,7 @@ public class RewriteConstantFolding extends HopRewriteRule
{
literal = new LiteralOp(true);
}
-
+
//replace binary operator with folded constant
if( literal != null )
{
@@ -163,8 +160,8 @@ public class RewriteConstantFolding extends HopRewriteRule
{
root = literal;
}
- }
-
+ }
+
//mark processed
root.setVisited();
@@ -195,7 +192,7 @@ public class RewriteConstantFolding extends HopRewriteRule
Dag<Lop> dag = new Dag<Lop>();
Recompiler.rClearLops(tmpWrite); //prevent lops reuse
Lop lops = tmpWrite.constructLops(); //reconstruct lops
- lops.addToDag( dag );
+ lops.addToDag( dag );
ArrayList<Instruction> inst = dag.getJobs(null, ConfigurationManager.getDMLConfig());
//execute instructions
http://git-wip-us.apache.org/repos/asf/systemml/blob/1d83cedb/src/main/java/org/apache/sysml/hops/rewrite/RewriteRemoveUnnecessaryBranches.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/rewrite/RewriteRemoveUnnecessaryBranches.java b/src/main/java/org/apache/sysml/hops/rewrite/RewriteRemoveUnnecessaryBranches.java
index 27e1dd3..6ada2d2 100644
--- a/src/main/java/org/apache/sysml/hops/rewrite/RewriteRemoveUnnecessaryBranches.java
+++ b/src/main/java/org/apache/sysml/hops/rewrite/RewriteRemoveUnnecessaryBranches.java
@@ -46,7 +46,7 @@ public class RewriteRemoveUnnecessaryBranches extends StatementBlockRewriteRule
if( sb instanceof IfStatementBlock )
{
IfStatementBlock isb = (IfStatementBlock) sb;
- Hop pred = isb.getPredicateHops();
+ Hop pred = isb.getPredicateHops().getInput().get(0);
//apply rewrite if literal op (constant value)
if( pred instanceof LiteralOp )
http://git-wip-us.apache.org/repos/asf/systemml/blob/1d83cedb/src/main/java/org/apache/sysml/parser/ConditionalPredicate.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/ConditionalPredicate.java b/src/main/java/org/apache/sysml/parser/ConditionalPredicate.java
index 1253391..2a11175 100644
--- a/src/main/java/org/apache/sysml/parser/ConditionalPredicate.java
+++ b/src/main/java/org/apache/sysml/parser/ConditionalPredicate.java
@@ -21,10 +21,7 @@ package org.apache.sysml.parser;
public class ConditionalPredicate implements ParseInfo
{
-
-
- Expression _expr;
-
+ private Expression _expr;
public ConditionalPredicate(Expression expr){
_expr = expr;
@@ -41,7 +38,6 @@ public class ConditionalPredicate implements ParseInfo
return _expr.toString();
}
-
public VariableSet variablesRead() {
VariableSet result = new VariableSet();
result.addVariables(_expr.variablesRead());
http://git-wip-us.apache.org/repos/asf/systemml/blob/1d83cedb/src/main/java/org/apache/sysml/parser/DMLTranslator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/DMLTranslator.java b/src/main/java/org/apache/sysml/parser/DMLTranslator.java
index 1dbdfdc..eaae69a 100644
--- a/src/main/java/org/apache/sysml/parser/DMLTranslator.java
+++ b/src/main/java/org/apache/sysml/parser/DMLTranslator.java
@@ -67,7 +67,6 @@ import org.apache.sysml.hops.recompile.Recompiler;
import org.apache.sysml.hops.rewrite.HopRewriteUtils;
import org.apache.sysml.hops.rewrite.ProgramRewriter;
import org.apache.sysml.lops.Lop;
-import org.apache.sysml.lops.LopProperties;
import org.apache.sysml.lops.LopsException;
import org.apache.sysml.lops.compile.Dag;
import org.apache.sysml.parser.Expression.BuiltinFunctionOp;
@@ -211,7 +210,7 @@ public class DMLTranslator
for (DataIdentifier id : fstmt.getOutputParams())
currentLiveOut.addVariable(id.getName(), id);
-
+
fsb._liveOut = currentLiveOut;
fsb.analyze(currentLiveIn, currentLiveOut);
}
@@ -505,18 +504,6 @@ public class DMLTranslator
// create while program block
WhileProgramBlock rtpb = new WhileProgramBlock(prog, pred_instruct);
- if (rtpb.getPredicateResultVar() == null) {
- // e.g case : WHILE(continue)
- if ( ((WhileStatementBlock) sb).get_predicateLops().getExecLocation() == LopProperties.ExecLocation.Data ) {
- String resultVar = ((WhileStatementBlock) sb).get_predicateLops().getOutputParameters().getLabel();
- rtpb.setPredicateResultVar( resultVar );
- }
- else {
- LOG.error(sb.printBlockErrorLocation() + "Error in translating the WHILE predicate.");
- throw new LopsException(sb.printBlockErrorLocation() + "Error in translating the WHILE predicate.");
-
- }
- }
//// process the body of the while statement block ////
WhileStatementBlock wsb = (WhileStatementBlock)sb;
@@ -541,9 +528,6 @@ public class DMLTranslator
retPB = rtpb;
- //post processing for generating missing instructions
- //retPB = verifyAndCorrectProgramBlock(sb.liveIn(), sb.liveOut(), sb._kill, retPB);
-
// add statement block
retPB.setStatementBlock(sb);
@@ -568,18 +552,6 @@ public class DMLTranslator
// create if program block
IfProgramBlock rtpb = new IfProgramBlock(prog, pred_instruct);
- if (rtpb.getPredicateResultVar() == null ) {
- // e.g case : If(continue)
- if ( ((IfStatementBlock) sb).get_predicateLops().getExecLocation() == LopProperties.ExecLocation.Data ) {
- String resultVar = ((IfStatementBlock) sb).get_predicateLops().getOutputParameters().getLabel();
- rtpb.setPredicateResultVar( resultVar );
- }
- else {
- LOG.error(sb.printBlockErrorLocation() + "Error in translating the IF predicate.");
- throw new LopsException(sb.printBlockErrorLocation() + "Error in translating the IF predicate.");
- }
- }
-
// process the body of the if statement block
IfStatementBlock isb = (IfStatementBlock)sb;
if (isb.getNumStatements() > 1){
@@ -621,7 +593,7 @@ public class DMLTranslator
// process For Statement - add runtime program blocks to program
// NOTE: applies to ForStatementBlock and ParForStatementBlock
else if (sb instanceof ForStatementBlock)
- {
+ {
ForStatementBlock fsb = (ForStatementBlock) sb;
// create DAGs for loop predicates
@@ -631,41 +603,36 @@ public class DMLTranslator
if( fsb.getFromHops()!=null )
fsb.getFromLops().addToDag(fromDag);
if( fsb.getToHops()!=null )
- fsb.getToLops().addToDag(toDag);
+ fsb.getToLops().addToDag(toDag);
if( fsb.getIncrementHops()!=null )
- fsb.getIncrementLops().addToDag(incrementDag);
-
- // create instructions for loop predicates
+ fsb.getIncrementLops().addToDag(incrementDag);
+
+ // create instructions for loop predicates
ArrayList<Instruction> fromInstructions = fromDag.getJobs(null, config);
ArrayList<Instruction> toInstructions = toDag.getJobs(null, config);
- ArrayList<Instruction> incrementInstructions = incrementDag.getJobs(null, config);
+ ArrayList<Instruction> incrementInstructions = incrementDag.getJobs(null, config);
// create for program block
String sbName = null;
ForProgramBlock rtpb = null;
IterablePredicate iterPred = fsb.getIterPredicate();
- String [] iterPredData= IterablePredicate.createIterablePredicateVariables(iterPred.getIterVar().getName(),
- fsb.getFromLops(), fsb.getToLops(), fsb.getIncrementLops());
if( sb instanceof ParForStatementBlock )
{
sbName = "ParForStatementBlock";
- rtpb = new ParForProgramBlock(prog, iterPredData,iterPred.getParForParams());
+ rtpb = new ParForProgramBlock(prog, iterPred.getIterVar().getName(), iterPred.getParForParams());
ParForProgramBlock pfrtpb = (ParForProgramBlock)rtpb;
pfrtpb.setResultVariables( ((ParForStatementBlock)sb).getResultVariables() );
pfrtpb.setStatementBlock((ParForStatementBlock)sb); //used for optimization and creating unscoped variables
}
- else //ForStatementBlock
- {
+ else {//ForStatementBlock
sbName = "ForStatementBlock";
- rtpb = new ForProgramBlock(prog, iterPredData);
+ rtpb = new ForProgramBlock(prog, iterPred.getIterVar().getName());
}
-
- rtpb.setFromInstructions( fromInstructions );
- rtpb.setToInstructions( toInstructions );
- rtpb.setIncrementInstructions( incrementInstructions );
- rtpb.setIterablePredicateVars( iterPredData );
+ rtpb.setFromInstructions(fromInstructions);
+ rtpb.setToInstructions(toInstructions);
+ rtpb.setIncrementInstructions(incrementInstructions);
// process the body of the for statement block
if (fsb.getNumStatements() > 1){
@@ -1213,7 +1180,7 @@ public class DMLTranslator
return;
}
- if (sb instanceof ForStatementBlock) { //NOTE: applies to ForStatementBlock and ParForStatementBlock
+ if (sb instanceof ForStatementBlock) { //incl ParForStatementBlock
constructHopsForForControlBlock((ForStatementBlock) sb);
return;
}
@@ -1223,7 +1190,6 @@ public class DMLTranslator
return;
}
-
HashMap<String, Hop> ids = new HashMap<String, Hop>();
ArrayList<Hop> output = new ArrayList<Hop>();
@@ -1671,6 +1637,12 @@ public class DMLTranslator
}
predicateHops = processExpression(cp.getPredicate(), null, _ids);
}
+
+ //create transient write to internal variable name on top of expression
+ //in order to ensure proper instruction generation
+ predicateHops = HopRewriteUtils.createDataOp(
+ ProgramBlock.PRED_VAR, predicateHops, DataOpTypes.TRANSIENTWRITE);
+
if (passedSB instanceof WhileStatementBlock)
((WhileStatementBlock)passedSB).setPredicateHops(predicateHops);
else if (passedSB instanceof IfStatementBlock)
@@ -1691,20 +1663,16 @@ public class DMLTranslator
throws ParseException
{
HashMap<String, Hop> _ids = new HashMap<String, Hop>();
-
+
// set iterable predicate
ForStatement fs = (ForStatement) fsb.getStatement(0);
IterablePredicate ip = fs.getIterablePredicate();
for(int i=0; i < 3; i++) {
- VariableSet varsRead = null;
- if (i==0)
- varsRead = ip.getFromExpr().variablesRead();
- else if (i==1)
- varsRead = ip.getToExpr().variablesRead();
- else if( ip.getIncrementExpr() != null )
- varsRead = ip.getIncrementExpr().variablesRead();
-
+ Expression expr = (i == 0) ? ip.getFromExpr() : (i == 1) ? ip.getToExpr() :
+ ( ip.getIncrementExpr() != null ) ? ip.getIncrementExpr() : null;
+ VariableSet varsRead = (expr != null) ? expr.variablesRead() : null;
+
if(varsRead != null) {
for (String varName : varsRead.getVariables().keySet()) {
@@ -1725,42 +1693,22 @@ public class DMLTranslator
}
}
+ //create transient write to internal variable name on top of expression
+ //in order to ensure proper instruction generation
+ Hop predicateHops = processTempIntExpression(expr, _ids);
+ if( predicateHops != null )
+ predicateHops = HopRewriteUtils.createDataOp(
+ ProgramBlock.PRED_VAR, predicateHops, DataOpTypes.TRANSIENTWRITE);
+
//construct hops for from, to, and increment expressions
- if(i==0)
- fsb.setFromHops( processTempIntExpression( ip.getFromExpr(), _ids ));
- else if(i==1)
- fsb.setToHops( processTempIntExpression( ip.getToExpr(), _ids ));
+ if( i == 0 )
+ fsb.setFromHops( predicateHops );
+ else if( i == 1 )
+ fsb.setToHops( predicateHops );
else if( ip.getIncrementExpr() != null )
- fsb.setIncrementHops( processTempIntExpression( ip.getIncrementExpr(), _ids ));
-
- }
-
- /*VariableSet varsRead = ip.variablesRead();
-
- for (String varName : varsRead.getVariables().keySet()) {
-
- DataIdentifier var = passedSB.liveIn().getVariable(varName);
- DataOp read = null;
- if (var == null) {
- LOG.error(var.printErrorLocation() + "variable '" + varName + "' is not available for iterable predicate");
- throw new ParseException(var.printErrorLocation() + "variable '" + varName + "' is not available for iterable predicate");
- }
- else {
- long actualDim1 = (var instanceof IndexedIdentifier) ? ((IndexedIdentifier)var).getOrigDim1() : var.getDim1();
- long actualDim2 = (var instanceof IndexedIdentifier) ? ((IndexedIdentifier)var).getOrigDim2() : var.getDim2();
- read = new DataOp(var.getName(), var.getDataType(), var.getValueType(), DataOpTypes.TRANSIENTREAD,
- null, actualDim1, actualDim2, var.getNnz(), var.getRowsInBlock(), var.getColumnsInBlock());
- read.setAllPositions(var.getBeginLine(), var.getBeginColumn(), var.getEndLine(), var.getEndColumn());
- }
- _ids.put(varName, read);
+ fsb.setIncrementHops( predicateHops );
}
-
- //construct hops for from, to, and increment expressions
- fsb.setFromHops( processTempIntExpression( ip.getFromExpr(), _ids ));
- fsb.setToHops( processTempIntExpression( ip.getToExpr(), _ids ));
- fsb.setIncrementHops( processTempIntExpression( ip.getIncrementExpr(), _ids ));*/
}
-
/**
* Construct Hops from parse tree : Process Expression in an assignment
@@ -1858,11 +1806,13 @@ public class DMLTranslator
private Hop processTempIntExpression( Expression source, HashMap<String, Hop> hops )
throws ParseException
{
+ if( source == null )
+ return null;
+
DataIdentifier tmpOut = createTarget();
tmpOut.setDataType(DataType.SCALAR);
tmpOut.setValueType(ValueType.INT);
source.setOutput(tmpOut);
-
return processExpression(source, tmpOut, hops );
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/1d83cedb/src/main/java/org/apache/sysml/parser/IfStatement.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/IfStatement.java b/src/main/java/org/apache/sysml/parser/IfStatement.java
index 0eea248..3c890af 100644
--- a/src/main/java/org/apache/sysml/parser/IfStatement.java
+++ b/src/main/java/org/apache/sysml/parser/IfStatement.java
@@ -25,7 +25,6 @@ import java.util.ArrayList;
public class IfStatement extends Statement
{
-
private ConditionalPredicate _predicate;
private ArrayList<StatementBlock> _ifBody;
private ArrayList<StatementBlock> _elseBody;
http://git-wip-us.apache.org/repos/asf/systemml/blob/1d83cedb/src/main/java/org/apache/sysml/parser/IfStatementBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/IfStatementBlock.java b/src/main/java/org/apache/sysml/parser/IfStatementBlock.java
index c172ffa..5410aca 100644
--- a/src/main/java/org/apache/sysml/parser/IfStatementBlock.java
+++ b/src/main/java/org/apache/sysml/parser/IfStatementBlock.java
@@ -34,7 +34,6 @@ import org.apache.sysml.parser.Expression.DataType;
public class IfStatementBlock extends StatementBlock
{
-
private Hop _predicateHops;
private Lop _predicateLops = null;
private boolean _requiresPredicateRecompile = false;
http://git-wip-us.apache.org/repos/asf/systemml/blob/1d83cedb/src/main/java/org/apache/sysml/parser/IterablePredicate.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/IterablePredicate.java b/src/main/java/org/apache/sysml/parser/IterablePredicate.java
index 19fcc4a..68274f5 100644
--- a/src/main/java/org/apache/sysml/parser/IterablePredicate.java
+++ b/src/main/java/org/apache/sysml/parser/IterablePredicate.java
@@ -22,13 +22,11 @@ package org.apache.sysml.parser;
import java.util.HashMap;
import org.antlr.v4.runtime.ParserRuleContext;
-import org.apache.sysml.lops.Lop;
public class IterablePredicate extends Expression
{
-
- private DataIdentifier _iterVar; // variable being iterated over
+ private DataIdentifier _iterVar; // variable being iterated over
private Expression _fromExpr;
private Expression _toExpr;
private Expression _incrementExpr;
@@ -40,95 +38,63 @@ public class IterablePredicate extends Expression
_fromExpr = fromExpr;
_toExpr = toExpr;
_incrementExpr = incrementExpr;
-
+
_parforParams = parForParamValues;
- this.setCtxValuesAndFilename(ctx, filename);
- }
-
- public String toString()
- {
- StringBuilder sb = new StringBuilder();
- sb.append( "(" );
- sb.append( _iterVar.getName() );
- sb.append(" in seq(");
- sb.append(_fromExpr.toString());
- sb.append(",");
- sb.append(_toExpr.toString());
- if (_incrementExpr != null) {
- sb.append(",");
- sb.append(_incrementExpr.toString());
- }
- sb.append( ")" );
- if (_parforParams != null && _parforParams.size() > 0){
- for (String key : _parforParams.keySet())
- {
- sb.append( "," );
- sb.append( key );
- sb.append( "=" );
- sb.append( _parforParams.get(key) );
- }
- }
- sb.append( ")" );
- return sb.toString();
+ setCtxValuesAndFilename(ctx, filename);
}
-
public VariableSet variablesRead() {
VariableSet result = new VariableSet();
- result.addVariables( _fromExpr.variablesRead() );
- result.addVariables( _toExpr.variablesRead() );
+ result.addVariables(_fromExpr.variablesRead());
+ result.addVariables(_toExpr.variablesRead());
if( _incrementExpr != null )
result.addVariables( _incrementExpr.variablesRead() );
-
return result;
}
-
-
+
public VariableSet variablesUpdated() {
VariableSet result = new VariableSet();
result.addVariable(_iterVar.getName(), _iterVar);
-
- return result;
+ return result;
}
-
+
@Override
public Expression rewriteExpression(String prefix) throws LanguageException {
//DataIdentifier newIterVar = (DataIdentifier)_iterVar.rewriteExpression(prefix);
//return new IterablePredicate(newIterVar, _from, _to, _increment);
LOG.error(this.printErrorLocation() + "rewriteExpression not supported for IterablePredicate");
throw new LanguageException(this.printErrorLocation() + "rewriteExpression not supported for IterablePredicate");
-
}
-
+
@Override
public void validateExpression(HashMap<String, DataIdentifier> ids, HashMap<String, ConstIdentifier> constVars, boolean conditional)
throws LanguageException
- {
+ {
//recursive validate
if (_iterVar instanceof FunctionCallIdentifier
|| _fromExpr instanceof FunctionCallIdentifier
|| _toExpr instanceof FunctionCallIdentifier
|| _incrementExpr instanceof FunctionCallIdentifier){
- raiseValidateError("user-defined function calls not supported for iterable predicates",
- false, LanguageException.LanguageErrorCodes.UNSUPPORTED_EXPRESSION);
+ raiseValidateError("user-defined function calls not supported for iterable predicates",
+ false, LanguageException.LanguageErrorCodes.UNSUPPORTED_EXPRESSION);
}
//1) VALIDATE ITERATION VARIABLE (index)
- // check the variable has either 1) not been defined already OR 2) defined as integer scalar
+ // check the variable has either 1) not been defined already OR 2) defined as integer scalar
if (ids.containsKey(_iterVar.getName())){
DataIdentifier otherDI = ids.get(_iterVar.getName());
if( otherDI.getDataType() != DataType.SCALAR || otherDI.getValueType() != ValueType.INT ){
- raiseValidateError("iterable predicate in for loop '" + _iterVar.getName() + "' must be a scalar integer", conditional);
- }
+ raiseValidateError("iterable predicate in for loop '" + _iterVar.getName()
+ + "' must be a scalar integer", conditional);
+ }
}
// set the values for DataIdentifer iterable variable
_iterVar.setIntProperties();
-
+
// add the iterVar to the variable set
ids.put(_iterVar.getName(), _iterVar);
-
//2) VALIDATE FOR PREDICATE in (from, to, increment)
// handle default increment if unspecified
if (_incrementExpr == null && _fromExpr instanceof ConstIdentifier && _toExpr instanceof ConstIdentifier) {
@@ -136,7 +102,7 @@ public class IterablePredicate extends Expression
ConstIdentifier cTo = (ConstIdentifier) _toExpr;
_incrementExpr = new IntIdentifier((cFrom.getLongValue() <= cTo.getLongValue()) ? 1 : -1, this);
}
-
+
//recursively validate the individual expression
_fromExpr.validateExpression(ids, constVars, conditional);
_toExpr.validateExpression(ids, constVars, conditional);
@@ -148,7 +114,7 @@ public class IterablePredicate extends Expression
checkNumericScalarOutput( _toExpr );
checkNumericScalarOutput( _incrementExpr );
}
-
+
public DataIdentifier getIterVar() {
return _iterVar;
}
@@ -189,22 +155,34 @@ public class IterablePredicate extends Expression
_parforParams = params;
}
- public static String[] createIterablePredicateVariables( String varName, Lop from, Lop to, Lop incr )
+ @Override
+ public String toString()
{
- String[] ret = new String[4]; //varname, from, to, incr
-
- ret[0] = varName;
-
- if( from.getType()==Lop.Type.Data )
- ret[1] = from.getOutputParameters().getLabel();
- if( to.getType()==Lop.Type.Data )
- ret[2] = to.getOutputParameters().getLabel();
- if( incr != null && incr.getType()==Lop.Type.Data )
- ret[3] = incr.getOutputParameters().getLabel();
-
- return ret;
+ StringBuilder sb = new StringBuilder();
+ sb.append( "(" );
+ sb.append( _iterVar.getName() );
+ sb.append(" in seq(");
+ sb.append(_fromExpr.toString());
+ sb.append(",");
+ sb.append(_toExpr.toString());
+ if (_incrementExpr != null) {
+ sb.append(",");
+ sb.append(_incrementExpr.toString());
+ }
+ sb.append( ")" );
+ if (_parforParams != null && _parforParams.size() > 0){
+ for (String key : _parforParams.keySet())
+ {
+ sb.append( "," );
+ sb.append( key );
+ sb.append( "=" );
+ sb.append( _parforParams.get(key) );
+ }
+ }
+ sb.append( ")" );
+ return sb.toString();
}
-
+
private void checkNumericScalarOutput( Expression expr )
throws LanguageException
{
@@ -214,12 +192,10 @@ public class IterablePredicate extends Expression
Identifier ident = expr.getOutput();
if( ident.getDataType() == DataType.MATRIX || ident.getDataType() == DataType.OBJECT ||
(ident.getDataType() == DataType.SCALAR && (ident.getValueType() == ValueType.BOOLEAN ||
- ident.getValueType() == ValueType.STRING ||
- ident.getValueType() == ValueType.OBJECT)) )
+ ident.getValueType() == ValueType.STRING || ident.getValueType() == ValueType.OBJECT)) )
{
LOG.error(this.printErrorLocation() + "expression in iterable predicate in for loop '" + expr.toString() + "' must return a numeric scalar");
throw new LanguageException(this.printErrorLocation() + "expression in iterable predicate in for loop '" + expr.toString() + "' must return a numeric scalar");
}
}
-
-} // end class
+}
http://git-wip-us.apache.org/repos/asf/systemml/blob/1d83cedb/src/main/java/org/apache/sysml/runtime/controlprogram/ForProgramBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/ForProgramBlock.java b/src/main/java/org/apache/sysml/runtime/controlprogram/ForProgramBlock.java
index 4a529a1..05b622d 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/ForProgramBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/ForProgramBlock.java
@@ -31,30 +31,25 @@ import org.apache.sysml.runtime.DMLScriptException;
import org.apache.sysml.runtime.controlprogram.caching.MatrixObject.UpdateType;
import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
import org.apache.sysml.runtime.instructions.Instruction;
-import org.apache.sysml.runtime.instructions.cp.Data;
import org.apache.sysml.runtime.instructions.cp.IntObject;
import org.apache.sysml.runtime.instructions.cp.ScalarObject;
-import org.apache.sysml.runtime.util.UtilFunctions;
import org.apache.sysml.yarn.DMLAppMasterUtils;
public class ForProgramBlock extends ProgramBlock
{
- protected ArrayList<Instruction> _fromInstructions;
- protected ArrayList<Instruction> _toInstructions;
- protected ArrayList<Instruction> _incrementInstructions;
+ protected ArrayList<Instruction> _fromInstructions;
+ protected ArrayList<Instruction> _toInstructions;
+ protected ArrayList<Instruction> _incrementInstructions;
+ protected ArrayList <Instruction> _exitInstructions;
+ protected ArrayList<ProgramBlock> _childBlocks;
+ protected String _iterPredVar;
- protected ArrayList <Instruction> _exitInstructions ;
- protected ArrayList<ProgramBlock> _childBlocks;
-
- protected String[] _iterablePredicateVars; //from,to,where constants/internal vars not captured via instructions
-
-
- public ForProgramBlock(Program prog, String[] iterPredVars) {
+ public ForProgramBlock(Program prog, String iterPredVar) {
super(prog);
_exitInstructions = new ArrayList<Instruction>();
_childBlocks = new ArrayList<ProgramBlock>();
- _iterablePredicateVars = iterPredVars;
+ _iterPredVar = iterPredVar;
}
public ArrayList<Instruction> getFromInstructions() {
@@ -101,30 +96,28 @@ public class ForProgramBlock extends ProgramBlock
_childBlocks = pbs;
}
- public String[] getIterablePredicateVars() {
- return _iterablePredicateVars;
+ public String getIterVar() {
+ return _iterPredVar;
}
- public void setIterablePredicateVars(String[] iterPredVars) {
- _iterablePredicateVars = iterPredVars;
+ public void setIterVar(String iterPredVar) {
+ _iterPredVar = iterPredVar;
}
@Override
public void execute(ExecutionContext ec)
throws DMLRuntimeException
{
- // add the iterable predicate variable to the variable set
- String iterVarName = _iterablePredicateVars[0];
-
// evaluate from, to, incr only once (assumption: known at for entry)
IntObject from = executePredicateInstructions( 1, _fromInstructions, ec );
IntObject to = executePredicateInstructions( 2, _toInstructions, ec );
- IntObject incr = (_incrementInstructions == null || _incrementInstructions.isEmpty()) && _iterablePredicateVars[3]==null ?
- new IntObject((from.getLongValue()<=to.getLongValue()) ? 1 : -1) :
- executePredicateInstructions( 3, _incrementInstructions, ec );
+ IntObject incr = (_incrementInstructions == null || _incrementInstructions.isEmpty()) ?
+ new IntObject((from.getLongValue()<=to.getLongValue()) ? 1 : -1) :
+ executePredicateInstructions( 3, _incrementInstructions, ec );
if ( incr.getLongValue() == 0 ) //would produce infinite loop
- throw new DMLRuntimeException(this.printBlockErrorLocation() + "Expression for increment of variable '" + iterVarName + "' must evaluate to a non-zero value.");
+ throw new DMLRuntimeException(printBlockErrorLocation() + "Expression for increment "
+ + "of variable '" + _iterPredVar + "' must evaluate to a non-zero value.");
// execute for loop
try
@@ -133,11 +126,11 @@ public class ForProgramBlock extends ProgramBlock
UpdateType[] flags = prepareUpdateInPlaceVariables(ec, _tid);
// run for loop body for each instance of predicate sequence
- SequenceIterator seqIter = new SequenceIterator(iterVarName, from, to, incr);
+ SequenceIterator seqIter = new SequenceIterator(_iterPredVar, from, to, incr);
for( IntObject iterVar : seqIter )
{
//set iteration variable
- ec.setVariable(iterVarName, iterVar);
+ ec.setVariable(_iterPredVar, iterVar);
//execute all child blocks
for(int i=0 ; i < this._childBlocks.size() ; i++) {
@@ -174,42 +167,30 @@ public class ForProgramBlock extends ProgramBlock
try
{
- if( _iterablePredicateVars[pos] != null )
- {
- //probe for scalar variables
- Data ldat = ec.getVariable( _iterablePredicateVars[pos] );
- if( ldat != null && ldat instanceof ScalarObject )
- tmp = (ScalarObject)ldat;
- else //handle literals
- tmp = new IntObject( UtilFunctions.parseToLong(_iterablePredicateVars[pos]) );
- }
- else
+ if( _sb != null )
{
- if( _sb!=null )
- {
- if( DMLScript.isActiveAM() ) //set program block specific remote memory
- DMLAppMasterUtils.setupProgramBlockRemoteMaxMemory(this);
-
- ForStatementBlock fsb = (ForStatementBlock)_sb;
- Hop predHops = null;
- boolean recompile = false;
- if (pos == 1){
- predHops = fsb.getFromHops();
- recompile = fsb.requiresFromRecompilation();
- }
- else if (pos == 2) {
- predHops = fsb.getToHops();
- recompile = fsb.requiresToRecompilation();
- }
- else if (pos == 3){
- predHops = fsb.getIncrementHops();
- recompile = fsb.requiresIncrementRecompilation();
- }
- tmp = (IntObject) executePredicate(instructions, predHops, recompile, ValueType.INT, ec);
+ if( DMLScript.isActiveAM() ) //set program block specific remote memory
+ DMLAppMasterUtils.setupProgramBlockRemoteMaxMemory(this);
+
+ ForStatementBlock fsb = (ForStatementBlock)_sb;
+ Hop predHops = null;
+ boolean recompile = false;
+ if (pos == 1){
+ predHops = fsb.getFromHops();
+ recompile = fsb.requiresFromRecompilation();
+ }
+ else if (pos == 2) {
+ predHops = fsb.getToHops();
+ recompile = fsb.requiresToRecompilation();
}
- else
- tmp = (IntObject) executePredicate(instructions, null, false, ValueType.INT, ec);
+ else if (pos == 3){
+ predHops = fsb.getIncrementHops();
+ recompile = fsb.requiresIncrementRecompilation();
+ }
+ tmp = (IntObject) executePredicate(instructions, predHops, recompile, ValueType.INT, ec);
}
+ else
+ tmp = (IntObject) executePredicate(instructions, null, false, ValueType.INT, ec);
}
catch(Exception ex)
{
http://git-wip-us.apache.org/repos/asf/systemml/blob/1d83cedb/src/main/java/org/apache/sysml/runtime/controlprogram/IfProgramBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/IfProgramBlock.java b/src/main/java/org/apache/sysml/runtime/controlprogram/IfProgramBlock.java
index cbb5ced..cecc8f7 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/IfProgramBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/IfProgramBlock.java
@@ -22,30 +22,19 @@ package org.apache.sysml.runtime.controlprogram;
import java.util.ArrayList;
import org.apache.sysml.api.DMLScript;
-import org.apache.sysml.hops.Hop;
import org.apache.sysml.parser.IfStatementBlock;
import org.apache.sysml.parser.Expression.ValueType;
import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.DMLScriptException;
import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
import org.apache.sysml.runtime.instructions.Instruction;
-import org.apache.sysml.runtime.instructions.Instruction.INSTRUCTION_TYPE;
import org.apache.sysml.runtime.instructions.cp.BooleanObject;
-import org.apache.sysml.runtime.instructions.cp.CPInstruction;
-import org.apache.sysml.runtime.instructions.cp.ComputationCPInstruction;
-import org.apache.sysml.runtime.instructions.cp.Data;
-import org.apache.sysml.runtime.instructions.cp.ScalarObject;
-import org.apache.sysml.runtime.instructions.cp.StringObject;
-import org.apache.sysml.runtime.instructions.cp.VariableCPInstruction;
-import org.apache.sysml.runtime.instructions.cp.CPInstruction.CPINSTRUCTION_TYPE;
import org.apache.sysml.yarn.DMLAppMasterUtils;
public class IfProgramBlock extends ProgramBlock
{
-
private ArrayList<Instruction> _predicate;
- private String _predicateResultVar;
private ArrayList <Instruction> _exitInstructions ;
private ArrayList<ProgramBlock> _childBlocksIfBody;
private ArrayList<ProgramBlock> _childBlocksElseBody;
@@ -55,9 +44,7 @@ public class IfProgramBlock extends ProgramBlock
_childBlocksIfBody = new ArrayList<ProgramBlock>();
_childBlocksElseBody = new ArrayList<ProgramBlock>();
-
_predicate = predicate;
- _predicateResultVar = findPredicateResultVar ();
_exitInstructions = new ArrayList<Instruction>();
}
@@ -101,22 +88,8 @@ public class IfProgramBlock extends ProgramBlock
return _predicate;
}
- public void setPredicate(ArrayList<Instruction> predicate)
- {
+ public void setPredicate(ArrayList<Instruction> predicate) {
_predicate = predicate;
-
- //update result var if non-empty predicate (otherwise,
- //do not overwrite varname predicate in predicateResultVar)
- if( _predicate != null && !_predicate.isEmpty() )
- _predicateResultVar = findPredicateResultVar();
- }
-
- public String getPredicateResultVar(){
- return _predicateResultVar;
- }
-
- public void setPredicateResultVar(String resultVar) {
- _predicateResultVar = resultVar;
}
public ArrayList<Instruction> getExitInstructions(){
@@ -184,47 +157,19 @@ public class IfProgramBlock extends ProgramBlock
BooleanObject result = null;
try
{
- if( _predicate!=null && !_predicate.isEmpty() )
- {
- if( _sb != null )
- {
- if( DMLScript.isActiveAM() ) //set program block specific remote memory
- DMLAppMasterUtils.setupProgramBlockRemoteMaxMemory(this);
-
- IfStatementBlock isb = (IfStatementBlock)_sb;
- Hop predicateOp = isb.getPredicateHops();
- boolean recompile = isb.requiresPredicateRecompilation();
- result = (BooleanObject) executePredicate(_predicate, predicateOp, recompile, ValueType.BOOLEAN, ec);
- }
- else
- result = (BooleanObject) executePredicate(_predicate, null, false, ValueType.BOOLEAN, ec);
- }
- else
+ if( _sb != null )
{
- //get result var
- ScalarObject scalarResult = null;
- Data resultData = ec.getVariable(_predicateResultVar);
- if ( resultData == null ) {
- //note: resultvar is a literal (can it be of any value type other than String, hence no literal/varname conflict)
- scalarResult = ec.getScalarInput(_predicateResultVar, ValueType.BOOLEAN, true);
- }
- else {
- scalarResult = ec.getScalarInput(_predicateResultVar, ValueType.BOOLEAN, false);
- }
-
- //check for invalid type String
- if (scalarResult instanceof StringObject)
- throw new DMLRuntimeException(this.printBlockErrorLocation() + "\nIf predicate variable "+ _predicateResultVar + " evaluated to string " + scalarResult + " which is not allowed for predicates in DML");
+ if( DMLScript.isActiveAM() ) //set program block specific remote memory
+ DMLAppMasterUtils.setupProgramBlockRemoteMaxMemory(this);
- //process result
- if( scalarResult instanceof BooleanObject )
- result = (BooleanObject)scalarResult;
- else
- result = new BooleanObject( scalarResult.getBooleanValue() ); //auto casting
+ IfStatementBlock isb = (IfStatementBlock)_sb;
+ result = (BooleanObject) executePredicate(_predicate, isb.getPredicateHops(),
+ isb.requiresPredicateRecompilation(), ValueType.BOOLEAN, ec);
}
+ else
+ result = (BooleanObject) executePredicate(_predicate, null, false, ValueType.BOOLEAN, ec);
}
- catch(Exception ex)
- {
+ catch(Exception ex) {
throw new DMLRuntimeException(this.printBlockErrorLocation() + "Failed to evaluate the IF predicate.", ex);
}
@@ -232,21 +177,7 @@ public class IfProgramBlock extends ProgramBlock
return result;
}
- private String findPredicateResultVar ( ) {
- String result = null;
- for ( Instruction si : _predicate ) {
- if ( si.getType() == INSTRUCTION_TYPE.CONTROL_PROGRAM && ((CPInstruction)si).getCPInstructionType() != CPINSTRUCTION_TYPE.Variable ) {
- result = ((ComputationCPInstruction) si).getOutputVariableName();
- }
- else if(si instanceof VariableCPInstruction && ((VariableCPInstruction)si).isVariableCastInstruction()){
- result = ((VariableCPInstruction)si).getOutputVariableName();
- }
- }
- return result;
- }
-
public String printBlockErrorLocation(){
return "ERROR: Runtime error in if program block generated from if statement block between lines " + _beginLine + " and " + _endLine + " -- ";
}
-
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/systemml/blob/1d83cedb/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java b/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java
index 9f225c6..1968c26 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java
@@ -300,7 +300,6 @@ public class ParForProgramBlock extends ForProgramBlock
//specifics used for optimization
protected long _numIterations = -1;
- protected String[] _iterablePredicateVarsOriginal = null;
//specifics used for data partitioning
protected LocalVariableMap _variablesDPOriginal = null;
@@ -342,10 +341,10 @@ public class ParForProgramBlock extends ForProgramBlock
_pwIDSeq = new IDSequence();
}
- public ParForProgramBlock(Program prog, String[] iterPredVars, HashMap<String,String> params)
+ public ParForProgramBlock(Program prog, String iterPredVar, HashMap<String,String> params)
throws DMLRuntimeException
{
- this( -1, prog, iterPredVars, params);
+ this( -1, prog, iterPredVar, params);
}
/**
@@ -358,9 +357,9 @@ public class ParForProgramBlock extends ForProgramBlock
* @param params map of parameters
* @throws DMLRuntimeException if DMLRuntimeException occurs
*/
- public ParForProgramBlock(int ID, Program prog, String[] iterPredVars, HashMap<String,String> params)
+ public ParForProgramBlock(int ID, Program prog, String iterPredVar, HashMap<String,String> params)
{
- super(prog, iterPredVars);
+ super(prog, iterPredVar);
//init internal flags according to DML config
initInternalConfigurations(ConfigurationManager.getDMLConfig());
@@ -551,38 +550,29 @@ public class ParForProgramBlock extends ForProgramBlock
throws DMLRuntimeException
{
ParForStatementBlock sb = (ParForStatementBlock)getStatementBlock();
-
- // add the iterable predicate variable to the variable set
- String iterVarName = _iterablePredicateVars[0];
// evaluate from, to, incr only once (assumption: known at for entry)
IntObject from = executePredicateInstructions( 1, _fromInstructions, ec );
IntObject to = executePredicateInstructions( 2, _toInstructions, ec );
- IntObject incr = (_incrementInstructions == null || _incrementInstructions.isEmpty()) && _iterablePredicateVars[3]==null ?
- new IntObject((from.getLongValue()<=to.getLongValue()) ? 1 : -1) :
- executePredicateInstructions( 3, _incrementInstructions, ec );
+ IntObject incr = (_incrementInstructions == null || _incrementInstructions.isEmpty()) ?
+ new IntObject((from.getLongValue()<=to.getLongValue()) ? 1 : -1) :
+ executePredicateInstructions( 3, _incrementInstructions, ec );
if ( incr.getLongValue() == 0 ) //would produce infinite loop
- throw new DMLRuntimeException(this.printBlockErrorLocation() + "Expression for increment of variable '" + iterVarName + "' must evaluate to a non-zero value.");
+ throw new DMLRuntimeException(this.printBlockErrorLocation() + "Expression for increment "
+ + "of variable '" + _iterPredVar + "' must evaluate to a non-zero value.");
//early exit on num iterations = zero
- if( computeNumIterations(from, to, incr) <= 0 )
+ _numIterations = computeNumIterations(from, to, incr);
+ if( _numIterations <= 0 )
return; //avoid unnecessary optimization/initialization
///////
//OPTIMIZATION of ParFOR body (incl all child parfor PBs)
///////
- if( _optMode != POptMode.NONE )
- {
- updateIterablePredicateVars( iterVarName, from, to, incr );
+ if( _optMode != POptMode.NONE ) {
OptimizationWrapper.setLogLevel(_optLogLevel); //set optimizer log level
- OptimizationWrapper.optimize( _optMode, sb, this, ec, _monitor ); //core optimize
-
- //take changed iterable predicate into account
- iterVarName = _iterablePredicateVars[0];
- from = executePredicateInstructions( 1, _fromInstructions, ec );
- to = executePredicateInstructions( 2, _toInstructions, ec );
- incr = executePredicateInstructions( 3, _incrementInstructions, ec );
+ OptimizationWrapper.optimize(_optMode, sb, this, ec, _monitor); //core optimize
}
///////
@@ -604,7 +594,7 @@ public class ParForProgramBlock extends ForProgramBlock
StatisticMonitor.putPFStat(_ID, Stat.PARFOR_INIT_DATA_T, time.stop());
// initialize iter var to form value
- IntObject iterVar = new IntObject(iterVarName, from.getLongValue() );
+ IntObject iterVar = new IntObject(_iterPredVar, from.getLongValue() );
///////
//begin PARALLEL EXECUTION of (PAR)FOR body
@@ -667,8 +657,8 @@ public class ParForProgramBlock extends ForProgramBlock
cleanupSharedVariables(ec, varState);
//set iteration var to TO value (+ increment) for FOR equivalence
- iterVar = new IntObject( iterVarName, to.getLongValue() ); //consistent with for
- ec.setVariable(iterVarName, iterVar);
+ iterVar = new IntObject(_iterPredVar, to.getLongValue()); //consistent with for
+ ec.setVariable(_iterPredVar, iterVar);
//ensure that subsequent program blocks never see partitioned data (invalid plans!)
//we can replace those variables, because partitioning only applied for read-only matrices
@@ -697,7 +687,6 @@ public class ParForProgramBlock extends ForProgramBlock
//release forced exectypes for fused dp/exec
if( _execMode == PExecMode.REMOTE_MR_DP || _execMode == PExecMode.REMOTE_SPARK_DP )
ProgramRecompiler.rFindAndRecompileIndexingHOP(sb, this, _colocatedDPMatrix, ec, false);
- resetIterablePredicateVars();
resetOptimizerFlags(); //after release, deletes dp_varnames
//execute exit instructions (usually empty)
@@ -1450,34 +1439,33 @@ public class ParForProgramBlock extends ForProgramBlock
{
TaskPartitioner tp = null;
- switch( _taskPartitioner )
- {
+ switch( _taskPartitioner ) {
case FIXED:
- tp = new TaskPartitionerFixedsize( _taskSize, _iterablePredicateVars[0],
- from, to, incr );
+ tp = new TaskPartitionerFixedsize(
+ _taskSize, _iterPredVar, from, to, incr);
break;
case NAIVE:
- tp = new TaskPartitionerNaive( _taskSize, _iterablePredicateVars[0],
- from, to, incr );
+ tp = new TaskPartitionerNaive(
+ _taskSize, _iterPredVar, from, to, incr);
break;
case STATIC:
- tp = new TaskPartitionerStatic( _taskSize, _numThreads, _iterablePredicateVars[0],
- from, to, incr );
+ tp = new TaskPartitionerStatic(
+ _taskSize, _numThreads, _iterPredVar, from, to, incr);
break;
case FACTORING:
- tp = new TaskPartitionerFactoring( _taskSize,_numThreads, _iterablePredicateVars[0],
- from, to, incr );
+ tp = new TaskPartitionerFactoring(
+ _taskSize,_numThreads, _iterPredVar, from, to, incr);
break;
case FACTORING_CMIN:
//for constrained factoring the tasksize is used as the minimum constraint
- tp = new TaskPartitionerFactoringCmin( _taskSize,_numThreads, _taskSize, _iterablePredicateVars[0],
- from, to, incr );
+ tp = new TaskPartitionerFactoringCmin(_taskSize,_numThreads,
+ _taskSize, _iterPredVar, from, to, incr);
break;
case FACTORING_CMAX:
//for constrained factoring the tasksize is used as the minimum constraint
- tp = new TaskPartitionerFactoringCmax( _taskSize,_numThreads, _taskSize, _iterablePredicateVars[0],
- from, to, incr );
+ tp = new TaskPartitionerFactoringCmax(_taskSize,_numThreads,
+ _taskSize, _iterPredVar, from, to, incr);
break;
default:
throw new DMLRuntimeException("Undefined task partitioner: '"+_taskPartitioner+"'.");
@@ -1828,27 +1816,6 @@ public class ParForProgramBlock extends ForProgramBlock
{
return (long)Math.ceil(((double)(to.getLongValue() - from.getLongValue() + 1)) / incr.getLongValue());
}
-
- private void updateIterablePredicateVars(String iterVarName, IntObject from, IntObject to, IntObject incr)
- {
- _numIterations = computeNumIterations(from, to, incr);
-
- //keep original iterable predicate
- _iterablePredicateVarsOriginal = new String[4];
- System.arraycopy(_iterablePredicateVars, 0, _iterablePredicateVarsOriginal, 0, 4);
-
- _iterablePredicateVars[0] = iterVarName;
- _iterablePredicateVars[1] = from.getStringValue();
- _iterablePredicateVars[2] = to.getStringValue();
- _iterablePredicateVars[3] = incr.getStringValue();
- }
-
- private void resetIterablePredicateVars()
- {
- //reset of modified for optimization (opt!=NONE)
- if( _iterablePredicateVarsOriginal!=null )
- System.arraycopy(_iterablePredicateVarsOriginal, 0, _iterablePredicateVars, 0, 4);
- }
/**
* NOTE: Only required for remote parfor. Hence, there is no need to transfer DMLConfig to
@@ -1978,15 +1945,14 @@ public class ParForProgramBlock extends ForProgramBlock
{
//reset all state that was set but is not guaranteed to be overwritten by optimizer
_variablesDPOriginal.removeAll();
- _iterablePredicateVarsOriginal = null;
- _colocatedDPMatrix = null;
- _replicationDP = WRITE_REPLICATION_FACTOR;
- _replicationExport = -1;
- _jvmReuse = true;
- _recompileMemoryBudget = -1;
+ _colocatedDPMatrix = null;
+ _replicationDP = WRITE_REPLICATION_FACTOR;
+ _replicationExport = -1;
+ _jvmReuse = true;
+ _recompileMemoryBudget = -1;
_enableRuntimePiggybacking = false;
- _variablesRP = null;
- _variablesECache = null;
+ _variablesRP = null;
+ _variablesECache = null;
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/1d83cedb/src/main/java/org/apache/sysml/runtime/controlprogram/ProgramBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/ProgramBlock.java b/src/main/java/org/apache/sysml/runtime/controlprogram/ProgramBlock.java
index f1b3d5d..b854ea0 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/ProgramBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/ProgramBlock.java
@@ -24,6 +24,7 @@ import java.util.ArrayList;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.jmlc.JMLCUtils;
import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.hops.Hop;
import org.apache.sysml.hops.OptimizerUtils;
@@ -39,13 +40,11 @@ import org.apache.sysml.runtime.controlprogram.caching.MatrixObject.UpdateType;
import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
import org.apache.sysml.runtime.instructions.Instruction;
import org.apache.sysml.runtime.instructions.cp.BooleanObject;
-import org.apache.sysml.runtime.instructions.cp.ComputationCPInstruction;
import org.apache.sysml.runtime.instructions.cp.Data;
import org.apache.sysml.runtime.instructions.cp.DoubleObject;
import org.apache.sysml.runtime.instructions.cp.IntObject;
import org.apache.sysml.runtime.instructions.cp.ScalarObject;
import org.apache.sysml.runtime.instructions.cp.StringObject;
-import org.apache.sysml.runtime.instructions.cp.VariableCPInstruction;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
import org.apache.sysml.utils.Statistics;
import org.apache.sysml.yarn.DMLAppMasterUtils;
@@ -53,6 +52,8 @@ import org.apache.sysml.yarn.DMLAppMasterUtils;
public class ProgramBlock implements ParseInfo
{
+ public static final String PRED_VAR = "__pred";
+
protected static final Log LOG = LogFactory.getLog(ProgramBlock.class.getName());
private static final boolean CHECK_MATRIX_SPARSITY = false;
@@ -188,6 +189,7 @@ public class ProgramBlock implements ParseInfo
{
tmp = Recompiler.recompileHopsDag(
hops, ec.getVariables(), null, false, true, _tid);
+ tmp = JMLCUtils.cleanupRuntimeInstructions(tmp, PRED_VAR);
}
if( DMLScript.STATISTICS ){
long t1 = System.nanoTime();
@@ -222,40 +224,15 @@ public class ProgramBlock implements ParseInfo
protected ScalarObject executePredicateInstructions(ArrayList<Instruction> inst, ValueType retType, ExecutionContext ec)
throws DMLRuntimeException
{
- ScalarObject ret = null;
- String retName = null;
-
- //execute all instructions
- for (int i = 0; i < inst.size(); i++)
- {
- //indexed access required due to debug mode
- Instruction currInst = inst.get(i);
- if( !isRemoveVariableInstruction(currInst) )
- {
- //execute instruction
- ec.updateDebugState(i);
- executeSingleInstruction(currInst, ec);
-
- //get last return name
- if(currInst instanceof ComputationCPInstruction )
- retName = ((ComputationCPInstruction) currInst).getOutputVariableName();
- else if(currInst instanceof VariableCPInstruction && ((VariableCPInstruction)currInst).getOutputVariableName()!=null)
- retName = ((VariableCPInstruction)currInst).getOutputVariableName();
- }
- }
-
- //get return value TODO: how do we differentiate literals and variables?
- ret = (ScalarObject) ec.getScalarInput(retName, retType, false);
-
- //execute rmvar instructions
- for (int i = 0; i < inst.size(); i++) {
- //indexed access required due to debug mode
- Instruction currInst = inst.get(i);
- if( isRemoveVariableInstruction(currInst) ) {
- ec.updateDebugState(i);
- executeSingleInstruction(currInst, ec);
- }
+ //execute all instructions (indexed access required due to debug mode)
+ int pos = 0;
+ for( Instruction currInst : inst ) {
+ ec.updateDebugState(pos++);
+ executeSingleInstruction(currInst, ec);
}
+
+ //get scalar return
+ ScalarObject ret = (ScalarObject) ec.getScalarInput(PRED_VAR, retType, false);
//check and correct scalar ret type (incl save double to int)
if( ret.getValueType() != retType )
@@ -268,6 +245,8 @@ public class ProgramBlock implements ParseInfo
//do nothing
}
+ //remove predicate variable
+ ec.removeVariable(PRED_VAR);
return ret;
}
@@ -367,12 +346,7 @@ public class ProgramBlock implements ParseInfo
mo.setUpdateType(flags[i]);
}
}
-
- private boolean isRemoveVariableInstruction(Instruction inst) {
- return ( inst instanceof VariableCPInstruction
- && ((VariableCPInstruction)inst).isRemoveVariable() );
- }
-
+
private void checkSparsity( Instruction lastInst, LocalVariableMap vars )
throws DMLRuntimeException
{
http://git-wip-us.apache.org/repos/asf/systemml/blob/1d83cedb/src/main/java/org/apache/sysml/runtime/controlprogram/WhileProgramBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/WhileProgramBlock.java b/src/main/java/org/apache/sysml/runtime/controlprogram/WhileProgramBlock.java
index 9a2b53c..6c8ed80 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/WhileProgramBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/WhileProgramBlock.java
@@ -30,30 +30,19 @@ import org.apache.sysml.runtime.DMLScriptException;
import org.apache.sysml.runtime.controlprogram.caching.MatrixObject.UpdateType;
import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
import org.apache.sysml.runtime.instructions.Instruction;
-import org.apache.sysml.runtime.instructions.Instruction.INSTRUCTION_TYPE;
import org.apache.sysml.runtime.instructions.cp.BooleanObject;
-import org.apache.sysml.runtime.instructions.cp.CPInstruction;
-import org.apache.sysml.runtime.instructions.cp.ComputationCPInstruction;
-import org.apache.sysml.runtime.instructions.cp.Data;
-import org.apache.sysml.runtime.instructions.cp.ScalarObject;
-import org.apache.sysml.runtime.instructions.cp.StringObject;
-import org.apache.sysml.runtime.instructions.cp.VariableCPInstruction;
-import org.apache.sysml.runtime.instructions.cp.CPInstruction.CPINSTRUCTION_TYPE;
import org.apache.sysml.yarn.DMLAppMasterUtils;
public class WhileProgramBlock extends ProgramBlock
{
-
private ArrayList<Instruction> _predicate;
- private String _predicateResultVar;
private ArrayList <Instruction> _exitInstructions ;
private ArrayList<ProgramBlock> _childBlocks;
public WhileProgramBlock(Program prog, ArrayList<Instruction> predicate) {
super(prog);
_predicate = predicate;
- _predicateResultVar = findPredicateResultVar ();
_exitInstructions = new ArrayList<Instruction>();
_childBlocks = new ArrayList<ProgramBlock>();
}
@@ -76,19 +65,6 @@ public class WhileProgramBlock extends ProgramBlock
public void setPredicate( ArrayList<Instruction> predicate ) {
_predicate = predicate;
-
- //update result var if non-empty predicate (otherwise,
- //do not overwrite varname predicate in predicateResultVar)
- if( _predicate != null && !_predicate.isEmpty() )
- _predicateResultVar = findPredicateResultVar();
- }
-
- public String getPredicateResultVar() {
- return _predicateResultVar;
- }
-
- public void setPredicateResultVar(String resultVar) {
- _predicateResultVar = resultVar;
}
public ArrayList<Instruction> getExitInstructions() {
@@ -101,44 +77,18 @@ public class WhileProgramBlock extends ProgramBlock
BooleanObject result = null;
try
{
- if( _predicate!=null && !_predicate.isEmpty() )
+ if( _sb!=null )
{
- if( _sb!=null )
- {
- if( DMLScript.isActiveAM() ) //set program block specific remote memory
- DMLAppMasterUtils.setupProgramBlockRemoteMaxMemory(this);
-
- WhileStatementBlock wsb = (WhileStatementBlock)_sb;
- Hop predicateOp = wsb.getPredicateHops();
- boolean recompile = wsb.requiresPredicateRecompilation();
- result = (BooleanObject) executePredicate(_predicate, predicateOp, recompile, ValueType.BOOLEAN, ec);
- }
- else
- result = (BooleanObject) executePredicate(_predicate, null, false, ValueType.BOOLEAN, ec);
- }
- else
- {
- //get result var
- ScalarObject scalarResult = null;
- Data resultData = ec.getVariable(_predicateResultVar);
- if ( resultData == null ) {
- //note: resultvar is a literal (can it be of any value type other than String, hence no literal/varname conflict)
- scalarResult = ec.getScalarInput(_predicateResultVar, ValueType.BOOLEAN, true);
- }
- else {
- scalarResult = ec.getScalarInput(_predicateResultVar, ValueType.BOOLEAN, false);
- }
+ if( DMLScript.isActiveAM() ) //set program block specific remote memory
+ DMLAppMasterUtils.setupProgramBlockRemoteMaxMemory(this);
- //check for invalid type String
- if (scalarResult instanceof StringObject)
- throw new DMLRuntimeException(this.printBlockErrorLocation() + "\nWhile predicate variable "+ _predicateResultVar + " evaluated to string " + scalarResult + " which is not allowed for predicates in DML");
-
- //process result
- if( scalarResult instanceof BooleanObject )
- result = (BooleanObject)scalarResult;
- else
- result = new BooleanObject( scalarResult.getBooleanValue() ); //auto casting
+ WhileStatementBlock wsb = (WhileStatementBlock)_sb;
+ Hop predicateOp = wsb.getPredicateHops();
+ boolean recompile = wsb.requiresPredicateRecompilation();
+ result = (BooleanObject) executePredicate(_predicate, predicateOp, recompile, ValueType.BOOLEAN, ec);
}
+ else
+ result = (BooleanObject) executePredicate(_predicate, null, false, ValueType.BOOLEAN, ec);
}
catch(Exception ex) {
throw new DMLRuntimeException(this.printBlockErrorLocation() + "Failed to evaluate the while predicate.", ex);
@@ -157,8 +107,7 @@ public class WhileProgramBlock extends ProgramBlock
UpdateType[] flags = prepareUpdateInPlaceVariables(ec, _tid);
//run loop body until predicate becomes false
- while( executePredicate(ec).getBooleanValue() )
- {
+ while( executePredicate(ec).getBooleanValue() ) {
//execute all child blocks
for (int i=0 ; i < _childBlocks.size() ; i++) {
ec.updateDebugState(i);
@@ -185,7 +134,7 @@ public class WhileProgramBlock extends ProgramBlock
throw new DMLRuntimeException(printBlockErrorLocation() + "Error executing while exit instructions.", e);
}
}
-
+
public ArrayList<ProgramBlock> getChildBlocks() {
return _childBlocks;
}
@@ -194,20 +143,6 @@ public class WhileProgramBlock extends ProgramBlock
_childBlocks = childs;
}
- private String findPredicateResultVar ( ) {
- String result = null;
- for ( Instruction si : _predicate ) {
- if ( si.getType() == INSTRUCTION_TYPE.CONTROL_PROGRAM && ((CPInstruction)si).getCPInstructionType() != CPINSTRUCTION_TYPE.Variable ) {
- result = ((ComputationCPInstruction) si).getOutputVariableName();
- }
- else if(si instanceof VariableCPInstruction && ((VariableCPInstruction)si).isVariableCastInstruction()){
- result = ((VariableCPInstruction)si).getOutputVariableName();
- }
- }
-
- return result;
- }
-
public String printBlockErrorLocation(){
return "ERROR: Runtime error in while program block generated from while statement block between lines " + _beginLine + " and " + _endLine + " -- ";
}