You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by lr...@apache.org on 2015/12/03 19:45:52 UTC
[21/78] [abbrv] [partial] incubator-systemml git commit: Move files
to new package folder structure
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/parser/DMLTranslator.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/parser/DMLTranslator.java b/src/main/java/com/ibm/bi/dml/parser/DMLTranslator.java
deleted file mode 100644
index ed6f955..0000000
--- a/src/main/java/com/ibm/bi/dml/parser/DMLTranslator.java
+++ /dev/null
@@ -1,2814 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.parser;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Iterator;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.conf.DMLConfig;
-import com.ibm.bi.dml.hops.AggBinaryOp;
-import com.ibm.bi.dml.hops.AggUnaryOp;
-import com.ibm.bi.dml.hops.BinaryOp;
-import com.ibm.bi.dml.hops.DataGenOp;
-import com.ibm.bi.dml.hops.DataOp;
-import com.ibm.bi.dml.hops.FunctionOp;
-import com.ibm.bi.dml.hops.FunctionOp.FunctionType;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.hops.Hop.AggOp;
-import com.ibm.bi.dml.hops.Hop.DataGenMethod;
-import com.ibm.bi.dml.hops.Hop.DataOpTypes;
-import com.ibm.bi.dml.hops.Hop.Direction;
-import com.ibm.bi.dml.hops.Hop.OpOp2;
-import com.ibm.bi.dml.hops.Hop.OpOp3;
-import com.ibm.bi.dml.hops.Hop.ParamBuiltinOp;
-import com.ibm.bi.dml.hops.Hop.ReOrgOp;
-import com.ibm.bi.dml.hops.HopsException;
-import com.ibm.bi.dml.hops.IndexingOp;
-import com.ibm.bi.dml.hops.LeftIndexingOp;
-import com.ibm.bi.dml.hops.LiteralOp;
-import com.ibm.bi.dml.hops.MemoTable;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.hops.ParameterizedBuiltinOp;
-import com.ibm.bi.dml.hops.ReorgOp;
-import com.ibm.bi.dml.hops.TernaryOp;
-import com.ibm.bi.dml.hops.UnaryOp;
-import com.ibm.bi.dml.hops.ipa.InterProceduralAnalysis;
-import com.ibm.bi.dml.hops.rewrite.ProgramRewriter;
-import com.ibm.bi.dml.hops.recompile.Recompiler;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.lops.LopsException;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.FormatType;
-import com.ibm.bi.dml.parser.Expression.ParameterizedBuiltinFunctionOp;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.parser.PrintStatement.PRINTTYPE;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-
-
-public class DMLTranslator
-{
-
- public static int DMLBlockSize = 1000;
-
- private static final Log LOG = LogFactory.getLog(DMLTranslator.class.getName());
- private DMLProgram _dmlProg = null;
-
-
- public DMLTranslator(DMLProgram dmlp)
- throws DMLRuntimeException
- {
- _dmlProg = dmlp;
-
- //each script sets its own block size, opt level etc
- DMLConfig conf = ConfigurationManager.getConfig();
- DMLTranslator.setDMLBlockSize( conf.getIntValue( DMLConfig.DEFAULT_BLOCK_SIZE ) );
- OptimizerUtils.setOptimizationLevel( conf.getIntValue(DMLConfig.OPTIMIZATION_LEVEL) );
- Recompiler.reinitRecompiler(); //reinit rewriter according to opt level flags
- }
-
- /**
- *
- * @param blocksize
- */
- public static void setDMLBlockSize(int blocksize) {
- DMLBlockSize = blocksize;
- }
-
- /**
- * Validate parse tree
- *
- * @throws LanguageException
- * @throws IOException
- */
- public void validateParseTree(DMLProgram dmlp)
- throws LanguageException, ParseException, IOException
- {
- //STEP1: Pre-processing steps for validate - e.g., prepare read-after-write meta data
- boolean fWriteRead = prepareReadAfterWrite(dmlp, new HashMap<String, DataIdentifier>());
-
- //STEP2: Actual Validate
- // handle functions in namespaces (current program has default namespace)
- for (String namespaceKey : dmlp.getNamespaces().keySet()){
-
- // for each function defined in the namespace
- for (String fname : dmlp.getFunctionStatementBlocks(namespaceKey).keySet()) {
- FunctionStatementBlock fblock = dmlp.getFunctionStatementBlock(namespaceKey,fname);
-
- HashMap<String, ConstIdentifier> constVars = new HashMap<String, ConstIdentifier>();
- VariableSet vs = new VariableSet();
-
- // add the input variables for the function to input variable list
- FunctionStatement fstmt = (FunctionStatement)fblock.getStatement(0);
- if (fblock.getNumStatements() > 1){
- LOG.error(fstmt.printErrorLocation() + "FunctionStatementBlock can only have 1 FunctionStatement");
- throw new LanguageException(fstmt.printErrorLocation() + "FunctionStatementBlock can only have 1 FunctionStatement");
- }
-
- for (DataIdentifier currVar : fstmt.getInputParams()) {
-
- if (currVar.getDataType() == DataType.SCALAR){
- currVar.setDimensions(0, 0);
- }
-
- vs.addVariable(currVar.getName(), currVar);
- }
- fblock.validate(dmlp, vs, constVars, false);
- }
-
- }
-
- // handle regular blocks -- "main" program
- VariableSet vs = new VariableSet();
- HashMap<String, ConstIdentifier> constVars = new HashMap<String, ConstIdentifier>();
- for (int i = 0; i < dmlp.getNumStatementBlocks(); i++) {
- StatementBlock sb = dmlp.getStatementBlock(i);
- vs = sb.validate(dmlp, vs, constVars, fWriteRead);
- constVars = sb.getConstOut();
- }
-
- //STEP3: Post-processing steps after validate - e.g., prepare read-after-write meta data
- if( fWriteRead )
- {
- //propagate size and datatypes into read
- prepareReadAfterWrite(dmlp, new HashMap<String, DataIdentifier>());
-
- //re-validate main program for datatype propagation
- vs = new VariableSet();
- constVars = new HashMap<String, ConstIdentifier>();
- for (int i = 0; i < dmlp.getNumStatementBlocks(); i++) {
- StatementBlock sb = dmlp.getStatementBlock(i);
- vs = sb.validate(dmlp, vs, constVars, fWriteRead);
- constVars = sb.getConstOut();
- }
- }
-
- return;
- }
-
- public void liveVariableAnalysis(DMLProgram dmlp) throws LanguageException {
-
- // for each namespace, handle function program blocks -- forward direction
- for (String namespaceKey : dmlp.getNamespaces().keySet()) {
- for (String fname: dmlp.getFunctionStatementBlocks(namespaceKey).keySet()) {
- FunctionStatementBlock fsb = dmlp.getFunctionStatementBlock(namespaceKey, fname);
- FunctionStatement fstmt = (FunctionStatement)fsb.getStatement(0);
-
- // perform function inlining
- fstmt.setBody(StatementBlock.mergeFunctionCalls(fstmt.getBody(), dmlp));
-
- VariableSet activeIn = new VariableSet();
- for (DataIdentifier id : fstmt.getInputParams()){
- activeIn.addVariable(id.getName(), id);
- }
- fsb.initializeforwardLV(activeIn);
- }
- }
-
- // for each namespace, handle function program blocks -- backward direction
- for (String namespaceKey : dmlp.getNamespaces().keySet()) {
- for (String fname: dmlp.getFunctionStatementBlocks(namespaceKey).keySet()) {
-
- // add output variables to liveout / activeout set
- FunctionStatementBlock fsb = dmlp.getFunctionStatementBlock(namespaceKey, fname);
- VariableSet currentLiveOut = new VariableSet();
- VariableSet currentLiveIn = new VariableSet();
- FunctionStatement fstmt = (FunctionStatement)fsb.getStatement(0);
-
- for (DataIdentifier id : fstmt.getInputParams())
- currentLiveIn.addVariable(id.getName(), id);
-
- for (DataIdentifier id : fstmt.getOutputParams())
- currentLiveOut.addVariable(id.getName(), id);
-
- fsb._liveOut = currentLiveOut;
- fsb.analyze(currentLiveIn, currentLiveOut);
- }
- }
-
-
- // handle regular program blocks
- VariableSet currentLiveOut = new VariableSet();
- VariableSet activeIn = new VariableSet();
-
- // handle function inlining
- dmlp.setStatementBlocks(StatementBlock.mergeFunctionCalls(dmlp.getStatementBlocks(), dmlp));
-
- for (int i = 0; i < dmlp.getNumStatementBlocks(); i++) {
- StatementBlock sb = dmlp.getStatementBlock(i);
- activeIn = sb.initializeforwardLV(activeIn);
- }
-
- if (dmlp.getNumStatementBlocks() > 0){
- StatementBlock lastSb = dmlp.getStatementBlock(dmlp.getNumStatementBlocks() - 1);
- lastSb._liveOut = new VariableSet();
- for (int i = dmlp.getNumStatementBlocks() - 1; i >= 0; i--) {
- StatementBlock sb = dmlp.getStatementBlock(i);
- currentLiveOut = sb.analyze(currentLiveOut);
- }
- }
- return;
-
- }
-
- /**
- * Construct Hops from parse tree
- *
- * @throws ParseException
- */
- public void constructHops(DMLProgram dmlp)
- throws ParseException, LanguageException
- {
- // Step 1: construct hops for all functions
- // for each namespace, handle function program blocks
- for (String namespaceKey : dmlp.getNamespaces().keySet()){
- for (String fname: dmlp.getFunctionStatementBlocks(namespaceKey).keySet()) {
- FunctionStatementBlock current = dmlp.getFunctionStatementBlock(namespaceKey, fname);
- constructHops(current);
- }
- }
-
- // Step 2: construct hops for main program
- // handle regular program blocks
- for (int i = 0; i < dmlp.getNumStatementBlocks(); i++) {
- StatementBlock current = dmlp.getStatementBlock(i);
- constructHops(current);
- }
- }
-
- /**
- *
- * @param dmlp
- * @throws ParseException
- * @throws LanguageException
- * @throws HopsException
- */
- public void rewriteHopsDAG(DMLProgram dmlp)
- throws ParseException, LanguageException, HopsException
- {
- //apply hop rewrites (static rewrites)
- ProgramRewriter rewriter = new ProgramRewriter(true, false);
- rewriter.rewriteProgramHopDAGs(dmlp);
- resetHopsDAGVisitStatus(dmlp);
-
- //propagate size information from main into functions (but conservatively)
- if( OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS ) {
- InterProceduralAnalysis ipa = new InterProceduralAnalysis();
- ipa.analyzeProgram(dmlp);
- resetHopsDAGVisitStatus(dmlp);
- }
-
- //apply hop rewrites (dynamic rewrites, after IPA)
- ProgramRewriter rewriter2 = new ProgramRewriter(false, true);
- rewriter2.rewriteProgramHopDAGs(dmlp);
- resetHopsDAGVisitStatus(dmlp);
-
- // Compute memory estimates for all the hops. These estimates are used
- // subsequently in various optimizations, e.g. CP vs. MR scheduling and parfor.
- refreshMemEstimates(dmlp);
- resetHopsDAGVisitStatus(dmlp);
- }
-
-
- public void constructLops(DMLProgram dmlp) throws ParseException, LanguageException, HopsException, LopsException {
-
- // for each namespace, handle function program blocks handle function
- for (String namespaceKey : dmlp.getNamespaces().keySet()){
- for (String fname: dmlp.getFunctionStatementBlocks(namespaceKey).keySet()) {
- FunctionStatementBlock current = dmlp.getFunctionStatementBlock(namespaceKey, fname);
- constructLops(current);
- }
- }
-
- // handle regular program blocks
- for (int i = 0; i < dmlp.getNumStatementBlocks(); i++) {
- StatementBlock current = dmlp.getStatementBlock(i);
- constructLops(current);
- }
- }
-
- /**
- *
- * @param sb
- * @throws HopsException
- * @throws LopsException
- */
- public void constructLops(StatementBlock sb)
- throws HopsException, LopsException
- {
- if (sb instanceof WhileStatementBlock)
- {
- WhileStatementBlock wsb = (WhileStatementBlock)sb;
- WhileStatement whileStmt = (WhileStatement)wsb.getStatement(0);
- ArrayList<StatementBlock> body = whileStmt.getBody();
-
- if (sb.get_hops() != null && !sb.get_hops().isEmpty()) {
- LOG.error(sb.printBlockErrorLocation() + "WhileStatementBlock should not have hops");
- throw new HopsException(sb.printBlockErrorLocation() + "WhileStatementBlock should not have hops");
- }
- // step through stmt blocks in while stmt body
- for (StatementBlock stmtBlock : body){
- constructLops(stmtBlock);
- }
-
- // handle while stmt predicate
- Lop l = wsb.getPredicateHops().constructLops();
- wsb.set_predicateLops(l);
- wsb.updatePredicateRecompilationFlag();
- }
-
- else if (sb instanceof IfStatementBlock)
- {
- IfStatementBlock isb = (IfStatementBlock) sb;
- IfStatement ifStmt = (IfStatement)isb.getStatement(0);
- ArrayList<StatementBlock> ifBody = ifStmt.getIfBody();
- ArrayList<StatementBlock> elseBody = ifStmt.getElseBody();
-
- if (sb.get_hops() != null && !sb.get_hops().isEmpty()){
- LOG.error(sb.printBlockErrorLocation() + "IfStatementBlock should not have hops");
- throw new HopsException(sb.printBlockErrorLocation() + "IfStatementBlock should not have hops");
- }
- // step through stmt blocks in if stmt ifBody
- for (StatementBlock stmtBlock : ifBody)
- constructLops(stmtBlock);
-
- // step through stmt blocks in if stmt elseBody
- for (StatementBlock stmtBlock : elseBody)
- constructLops(stmtBlock);
-
- // handle if stmt predicate
- Lop l = isb.getPredicateHops().constructLops();
- isb.set_predicateLops(l);
- isb.updatePredicateRecompilationFlag();
- }
-
- else if (sb instanceof ForStatementBlock) //NOTE: applies to ForStatementBlock and ParForStatementBlock
- {
- ForStatementBlock fsb = (ForStatementBlock) sb;
- ForStatement fs = (ForStatement)sb.getStatement(0);
- ArrayList<StatementBlock> body = fs.getBody();
-
- if (sb.get_hops() != null && !sb.get_hops().isEmpty() ) {
- LOG.error(sb.printBlockErrorLocation() + "ForStatementBlock should not have hops");
- throw new HopsException(sb.printBlockErrorLocation() + "ForStatementBlock should not have hops");
- }
- // step through stmt blocks in FOR stmt body
- for (StatementBlock stmtBlock : body)
- constructLops(stmtBlock);
-
- // handle for stmt predicate
- if (fsb.getFromHops() != null){
- Lop llobs = fsb.getFromHops().constructLops();
- fsb.setFromLops(llobs);
- }
- if (fsb.getToHops() != null){
- Lop llobs = fsb.getToHops().constructLops();
- fsb.setToLops(llobs);
- }
- if (fsb.getIncrementHops() != null){
- Lop llobs = fsb.getIncrementHops().constructLops();
- fsb.setIncrementLops(llobs);
- }
- fsb.updatePredicateRecompilationFlags();
- }
- else if (sb instanceof FunctionStatementBlock){
- FunctionStatement functStmt = (FunctionStatement)sb.getStatement(0);
- ArrayList<StatementBlock> body = functStmt.getBody();
-
- if (sb.get_hops() != null && !sb.get_hops().isEmpty()) {
- LOG.error(sb.printBlockErrorLocation() + "FunctionStatementBlock should not have hops");
- throw new HopsException(sb.printBlockErrorLocation() + "FunctionStatementBlock should not have hops");
- }
- // step through stmt blocks in while stmt body
- for (StatementBlock stmtBlock : body){
- constructLops(stmtBlock);
- }
- }
-
- // handle default case for regular StatementBlock
- else {
-
- if (sb.get_hops() == null)
- sb.set_hops(new ArrayList<Hop>());
-
- ArrayList<Lop> lops = new ArrayList<Lop>();
- for (Hop hop : sb.get_hops()) {
- lops.add(hop.constructLops());
- }
- sb.setLops(lops);
- sb.updateRecompilationFlag();
- }
-
- } // end method
-
-
- public void printLops(DMLProgram dmlp) throws ParseException, LanguageException, HopsException, LopsException {
- if (LOG.isDebugEnabled()){
- // for each namespace, handle function program blocks
- for (String namespaceKey : dmlp.getNamespaces().keySet()){
- for (String fname : dmlp.getFunctionStatementBlocks(namespaceKey).keySet()){
- FunctionStatementBlock fsblock = dmlp.getFunctionStatementBlock(namespaceKey,fname);
- printLops(fsblock);
- }
- }
-
- for (int i = 0; i < dmlp.getNumStatementBlocks(); i++) {
- StatementBlock current = dmlp.getStatementBlock(i);
- printLops(current);
- }
- }
- }
-
- public void printLops(StatementBlock current) throws ParseException, HopsException, LopsException {
- if (LOG.isDebugEnabled()){
- ArrayList<Lop> lopsDAG = current.getLops();
-
- LOG.debug("\n********************** LOPS DAG FOR BLOCK *******************");
-
- if (current instanceof FunctionStatementBlock) {
- if (current.getNumStatements() > 1)
- LOG.debug("Function statement block has more than 1 stmt");
- FunctionStatement fstmt = (FunctionStatement)current.getStatement(0);
- for (StatementBlock child : fstmt.getBody()){
- printLops(child);
- }
- }
-
- if (current instanceof WhileStatementBlock) {
-
- // print predicate lops
- WhileStatementBlock wstb = (WhileStatementBlock) current;
- Hop predicateHops = ((WhileStatementBlock) current).getPredicateHops();
- LOG.debug("\n********************** PREDICATE LOPS *******************");
- Lop predicateLops = predicateHops.getLops();
- if (predicateLops == null)
- predicateLops = predicateHops.constructLops();
- predicateLops.printMe();
-
- if (wstb.getNumStatements() > 1){
- LOG.error(wstb.printBlockErrorLocation() + "WhileStatementBlock has more than 1 statement");
- throw new HopsException(wstb.printBlockErrorLocation() + "WhileStatementBlock has more than 1 statement");
- }
- WhileStatement ws = (WhileStatement)wstb.getStatement(0);
-
- for (StatementBlock sb : ws.getBody()){
- printLops(sb);
- }
- }
-
- if (current instanceof IfStatementBlock) {
-
- // print predicate lops
- IfStatementBlock istb = (IfStatementBlock) current;
- Hop predicateHops = ((IfStatementBlock) current).getPredicateHops();
- LOG.debug("\n********************** PREDICATE LOPS *******************");
- Lop predicateLops = predicateHops.getLops();
- if (predicateLops == null)
- predicateLops = predicateHops.constructLops();
- predicateLops.printMe();
-
- if (istb.getNumStatements() > 1){
- LOG.error(istb.printBlockErrorLocation() + "IfStatmentBlock has more than 1 statement");
- throw new HopsException(istb.printBlockErrorLocation() + "IfStatmentBlock has more than 1 statement");
- }
- IfStatement is = (IfStatement)istb.getStatement(0);
-
- LOG.debug("\n**** LOPS DAG FOR IF BODY ****");
- for (StatementBlock sb : is.getIfBody()){
- printLops(sb);
- }
- if ( !is.getElseBody().isEmpty() ){
- LOG.debug("\n**** LOPS DAG FOR IF BODY ****");
- for (StatementBlock sb : is.getElseBody()){
- printLops(sb);
- }
- }
- }
-
- if (current instanceof ForStatementBlock) {
-
- // print predicate lops
- ForStatementBlock fsb = (ForStatementBlock) current;
- LOG.debug("\n********************** PREDICATE LOPS *******************");
- if( fsb.getFromHops() != null ){
- LOG.debug("FROM:");
- Lop llops = fsb.getFromLops();
- if( llops == null )
- llops = fsb.getFromHops().constructLops();
- llops.printMe();
- }
- if( fsb.getToHops() != null ){
- LOG.debug("TO:");
- Lop llops = fsb.getToLops();
- if( llops == null )
- llops = fsb.getToHops().constructLops();
- llops.printMe();
- }
- if( fsb.getIncrementHops() != null ){
- LOG.debug("INCREMENT:");
- Lop llops = fsb.getIncrementLops();
- if( llops == null )
- llops = fsb.getIncrementHops().constructLops();
- llops.printMe();
- }
-
- if (fsb.getNumStatements() > 1){
- LOG.error(fsb.printBlockErrorLocation() + "ForStatementBlock has more than 1 statement");
- throw new HopsException(fsb.printBlockErrorLocation() + "ForStatementBlock has more than 1 statement");
- }
- ForStatement ws = (ForStatement)fsb.getStatement(0);
-
- for (StatementBlock sb : ws.getBody()){
- printLops(sb);
- }
- }
-
- if (lopsDAG != null && !lopsDAG.isEmpty() ) {
- Iterator<Lop> iter = lopsDAG.iterator();
- while (iter.hasNext()) {
- LOG.debug("\n********************** OUTPUT LOPS *******************");
- iter.next().printMe();
- }
- }
- }
- }
-
-
- public void printHops(DMLProgram dmlp) throws ParseException, LanguageException, HopsException {
- if (LOG.isDebugEnabled()) {
- // for each namespace, handle function program blocks
- for (String namespaceKey : dmlp.getNamespaces().keySet()){
- for (String fname : dmlp.getFunctionStatementBlocks(namespaceKey).keySet()){
- FunctionStatementBlock fsblock = dmlp.getFunctionStatementBlock(namespaceKey,fname);
- printHops(fsblock);
- }
- }
-
- // hand
- for (int i = 0; i < dmlp.getNumStatementBlocks(); i++) {
- StatementBlock current = dmlp.getStatementBlock(i);
- printHops(current);
- }
- }
- }
-
- public void printHops(StatementBlock current) throws ParseException, HopsException {
- if (LOG.isDebugEnabled()) {
- ArrayList<Hop> hopsDAG = current.get_hops();
- LOG.debug("\n********************** HOPS DAG FOR BLOCK *******************");
-
- if (current instanceof FunctionStatementBlock) {
- if (current.getNumStatements() > 1)
- LOG.debug("Function statement block has more than 1 stmt");
- FunctionStatement fstmt = (FunctionStatement)current.getStatement(0);
- for (StatementBlock child : fstmt.getBody()){
- printHops(child);
- }
- }
-
- if (current instanceof WhileStatementBlock) {
-
- // print predicate hops
- WhileStatementBlock wstb = (WhileStatementBlock) current;
- Hop predicateHops = wstb.getPredicateHops();
- LOG.debug("\n********************** PREDICATE HOPS *******************");
- predicateHops.printMe();
-
- if (wstb.getNumStatements() > 1)
- LOG.debug("While statement block has more than 1 stmt");
- WhileStatement ws = (WhileStatement)wstb.getStatement(0);
-
- for (StatementBlock sb : ws.getBody()){
- printHops(sb);
- }
- }
-
- if (current instanceof IfStatementBlock) {
-
- // print predicate hops
- IfStatementBlock istb = (IfStatementBlock) current;
- Hop predicateHops = istb.getPredicateHops();
- LOG.debug("\n********************** PREDICATE HOPS *******************");
- predicateHops.printMe();
-
-
- if (istb.getNumStatements() > 1)
- LOG.debug("If statement block has more than 1 stmt");
- IfStatement is = (IfStatement)istb.getStatement(0);
-
- for (StatementBlock sb : is.getIfBody()){
- printHops(sb);
- }
-
- for (StatementBlock sb : is.getElseBody()){
- printHops(sb);
- }
- }
-
-
- if (current instanceof ForStatementBlock) {
-
- // print predicate hops
- ForStatementBlock fsb = (ForStatementBlock) current;
- LOG.debug("\n********************** PREDICATE HOPS *******************");
- if (fsb.getFromHops() != null) fsb.getFromHops().printMe();
- if (fsb.getToHops() != null) fsb.getToHops().printMe();
- if (fsb.getIncrementHops() != null) fsb.getIncrementHops().printMe();
-
- if (fsb.getNumStatements() > 1)
- LOG.debug("For statement block has more than 1 stmt");
- ForStatement ws = (ForStatement)fsb.getStatement(0);
-
- for (StatementBlock sb : ws.getBody()){
- printHops(sb);
- }
- }
-
- if (hopsDAG != null && !hopsDAG.isEmpty()) {
- // hopsDAG.iterator().next().printMe();
- Iterator<Hop> iter = hopsDAG.iterator();
- while (iter.hasNext()) {
- LOG.debug("\n********************** OUTPUT HOPS *******************");
- iter.next().printMe();
- }
- }
- }
- }
-
- public void refreshMemEstimates(DMLProgram dmlp) throws ParseException, LanguageException, HopsException {
-
- // for each namespace, handle function program blocks -- forward direction
- for (String namespaceKey : dmlp.getNamespaces().keySet()){
- for (String fname : dmlp.getFunctionStatementBlocks(namespaceKey).keySet()){
- FunctionStatementBlock fsblock = dmlp.getFunctionStatementBlock(namespaceKey, fname);
- refreshMemEstimates(fsblock);
- }
- }
-
- // handle statement blocks in "main" method
- for (int i = 0; i < dmlp.getNumStatementBlocks(); i++) {
- StatementBlock current = dmlp.getStatementBlock(i);
- refreshMemEstimates(current);
- }
- }
-
- public void refreshMemEstimates(StatementBlock current) throws ParseException, HopsException {
-
- MemoTable memo = new MemoTable();
- ArrayList<Hop> hopsDAG = current.get_hops();
- if (hopsDAG != null && !hopsDAG.isEmpty()) {
- Iterator<Hop> iter = hopsDAG.iterator();
- while (iter.hasNext()) {
- iter.next().refreshMemEstimates(memo);
- }
- }
-
- if (current instanceof FunctionStatementBlock) {
-
- FunctionStatement fstmt = (FunctionStatement)current.getStatement(0);
- for (StatementBlock sb : fstmt.getBody()){
- refreshMemEstimates(sb);
- }
- }
-
- if (current instanceof WhileStatementBlock) {
- // handle predicate
- WhileStatementBlock wstb = (WhileStatementBlock) current;
- wstb.getPredicateHops().refreshMemEstimates(new MemoTable());
-
- if (wstb.getNumStatements() > 1)
- LOG.debug("While statement block has more than 1 stmt");
- WhileStatement ws = (WhileStatement)wstb.getStatement(0);
-
- for (StatementBlock sb : ws.getBody()){
- refreshMemEstimates(sb);
- }
- }
-
- if (current instanceof IfStatementBlock) {
- // handle predicate
- IfStatementBlock istb = (IfStatementBlock) current;
- istb.getPredicateHops().refreshMemEstimates(new MemoTable());
-
- if (istb.getNumStatements() > 1)
- LOG.debug("If statement block has more than 1 stmt");
- IfStatement is = (IfStatement)istb.getStatement(0);
-
- for (StatementBlock sb : is.getIfBody()){
- refreshMemEstimates(sb);
- }
- for (StatementBlock sb : is.getElseBody()){
- refreshMemEstimates(sb);
- }
- }
-
- if (current instanceof ForStatementBlock) {
- // handle predicate
- ForStatementBlock fsb = (ForStatementBlock) current;
- if (fsb.getFromHops() != null)
- fsb.getFromHops().refreshMemEstimates(new MemoTable());
- if (fsb.getToHops() != null)
- fsb.getToHops().refreshMemEstimates(new MemoTable());
- if (fsb.getIncrementHops() != null)
- fsb.getIncrementHops().refreshMemEstimates(new MemoTable());
-
- if (fsb.getNumStatements() > 1)
- LOG.debug("For statement block has more than 1 stmt");
- ForStatement ws = (ForStatement)fsb.getStatement(0);
-
- for (StatementBlock sb : ws.getBody()){
- refreshMemEstimates(sb);
- }
- }
- }
-
- public static void resetHopsDAGVisitStatus(DMLProgram dmlp) throws ParseException, LanguageException, HopsException {
-
- // for each namespace, handle function program blocks -- forward direction
- for (String namespaceKey : dmlp.getNamespaces().keySet()){
- for (String fname : dmlp.getFunctionStatementBlocks(namespaceKey).keySet()){
- FunctionStatementBlock fsblock = dmlp.getFunctionStatementBlock(namespaceKey, fname);
- resetHopsDAGVisitStatus(fsblock);
- }
- }
-
- // handle statement blocks in "main" method
- for (int i = 0; i < dmlp.getNumStatementBlocks(); i++) {
- StatementBlock current = dmlp.getStatementBlock(i);
- resetHopsDAGVisitStatus(current);
- }
- }
-
- public static void resetHopsDAGVisitStatus(StatementBlock current) throws ParseException, HopsException {
-
- ArrayList<Hop> hopsDAG = current.get_hops();
- if (hopsDAG != null && !hopsDAG.isEmpty() ) {
- Hop.resetVisitStatus(hopsDAG);
- }
-
- if (current instanceof FunctionStatementBlock) {
-
- FunctionStatement fstmt = (FunctionStatement)current.getStatement(0);
- for (StatementBlock sb : fstmt.getBody()){
- resetHopsDAGVisitStatus(sb);
- }
- }
-
- if (current instanceof WhileStatementBlock) {
- // handle predicate
- WhileStatementBlock wstb = (WhileStatementBlock) current;
- wstb.getPredicateHops().resetVisitStatus();
-
- if (wstb.getNumStatements() > 1)
- LOG.debug("While stmt block has more than 1 stmt");
- WhileStatement ws = (WhileStatement)wstb.getStatement(0);
-
- for (StatementBlock sb : ws.getBody()){
- resetHopsDAGVisitStatus(sb);
- }
- }
-
- if (current instanceof IfStatementBlock) {
- // handle predicate
- IfStatementBlock istb = (IfStatementBlock) current;
- istb.getPredicateHops().resetVisitStatus();
-
- if (istb.getNumStatements() > 1)
- LOG.debug("If statement block has more than 1 stmt");
- IfStatement is = (IfStatement)istb.getStatement(0);
-
- for (StatementBlock sb : is.getIfBody()){
- resetHopsDAGVisitStatus(sb);
- }
- for (StatementBlock sb : is.getElseBody()){
- resetHopsDAGVisitStatus(sb);
- }
- }
-
- if (current instanceof ForStatementBlock) {
- // handle predicate
- ForStatementBlock fsb = (ForStatementBlock) current;
- if (fsb.getFromHops() != null)
- fsb.getFromHops().resetVisitStatus();
- if (fsb.getToHops() != null)
- fsb.getToHops().resetVisitStatus();
- if (fsb.getIncrementHops() != null)
- fsb.getIncrementHops().resetVisitStatus();
-
- if (fsb.getNumStatements() > 1)
- LOG.debug("For statment block has more than 1 stmt");
- ForStatement ws = (ForStatement)fsb.getStatement(0);
-
- for (StatementBlock sb : ws.getBody()){
- resetHopsDAGVisitStatus(sb);
- }
- }
- }
-
- public void resetLopsDAGVisitStatus(DMLProgram dmlp) throws HopsException, LanguageException {
-
- // for each namespace, handle function program blocks
- for (String namespaceKey : dmlp.getNamespaces().keySet()){
- for (String fname : dmlp.getFunctionStatementBlocks(namespaceKey).keySet()){
- FunctionStatementBlock fsblock = dmlp.getFunctionStatementBlock(namespaceKey, fname);
- resetLopsDAGVisitStatus(fsblock);
- }
- }
-
- for (int i = 0; i < dmlp.getNumStatementBlocks(); i++) {
- StatementBlock current = dmlp.getStatementBlock(i);
- resetLopsDAGVisitStatus(current);
- }
- }
-
- public void resetLopsDAGVisitStatus(StatementBlock current) throws HopsException {
-
- ArrayList<Hop> hopsDAG = current.get_hops();
-
- if (hopsDAG != null && !hopsDAG.isEmpty() ) {
- Iterator<Hop> iter = hopsDAG.iterator();
- while (iter.hasNext()){
- Hop currentHop = iter.next();
- currentHop.getLops().resetVisitStatus();
- }
- }
-
- if (current instanceof FunctionStatementBlock) {
- FunctionStatementBlock fsb = (FunctionStatementBlock) current;
- FunctionStatement fs = (FunctionStatement)fsb.getStatement(0);
-
- for (StatementBlock sb : fs.getBody()){
- resetLopsDAGVisitStatus(sb);
- }
- }
-
-
- if (current instanceof WhileStatementBlock) {
- WhileStatementBlock wstb = (WhileStatementBlock) current;
- wstb.get_predicateLops().resetVisitStatus();
- if (wstb.getNumStatements() > 1)
- LOG.debug("While statement block has more than 1 stmt");
- WhileStatement ws = (WhileStatement)wstb.getStatement(0);
-
- for (StatementBlock sb : ws.getBody()){
- resetLopsDAGVisitStatus(sb);
- }
- }
-
- if (current instanceof IfStatementBlock) {
- IfStatementBlock istb = (IfStatementBlock) current;
- istb.get_predicateLops().resetVisitStatus();
- if (istb.getNumStatements() > 1)
- LOG.debug("If statement block has more than 1 stmt");
- IfStatement is = (IfStatement)istb.getStatement(0);
-
- for (StatementBlock sb : is.getIfBody()){
- resetLopsDAGVisitStatus(sb);
- }
-
- for (StatementBlock sb : is.getElseBody()){
- resetLopsDAGVisitStatus(sb);
- }
- }
-
- if (current instanceof ForStatementBlock) {
- ForStatementBlock fsb = (ForStatementBlock) current;
-
- if (fsb.getFromLops() != null)
- fsb.getFromLops().resetVisitStatus();
- if (fsb.getToLops() != null)
- fsb.getToLops().resetVisitStatus();
- if (fsb.getIncrementLops() != null)
- fsb.getIncrementLops().resetVisitStatus();
-
- if (fsb.getNumStatements() > 1)
- LOG.debug("For statement block has more than 1 stmt");
- ForStatement ws = (ForStatement)fsb.getStatement(0);
-
- for (StatementBlock sb : ws.getBody()){
- resetLopsDAGVisitStatus(sb);
- }
- }
- }
-
-
- public void constructHops(StatementBlock sb)
- throws ParseException, LanguageException {
-
- if (sb instanceof WhileStatementBlock) {
- constructHopsForWhileControlBlock((WhileStatementBlock) sb);
- return;
- }
-
- if (sb instanceof IfStatementBlock) {
- constructHopsForIfControlBlock((IfStatementBlock) sb);
- return;
- }
-
- if (sb instanceof ForStatementBlock) { //NOTE: applies to ForStatementBlock and ParForStatementBlock
- constructHopsForForControlBlock((ForStatementBlock) sb);
- return;
- }
-
- if (sb instanceof FunctionStatementBlock) {
- constructHopsForFunctionControlBlock((FunctionStatementBlock) sb);
- return;
- }
-
-
- HashMap<String, Hop> ids = new HashMap<String, Hop>();
- ArrayList<Hop> output = new ArrayList<Hop>();
-
- VariableSet liveIn = sb.liveIn();
- VariableSet liveOut = sb.liveOut();
- VariableSet updated = sb._updated;
- VariableSet gen = sb._gen;
- VariableSet updatedLiveOut = new VariableSet();
-
- // handle liveout variables that are updated --> target identifiers for Assignment
- HashMap<String, Integer> liveOutToTemp = new HashMap<String, Integer>();
- for (int i = 0; i < sb.getNumStatements(); i++) {
- Statement current = sb.getStatement(i);
-
- if (current instanceof AssignmentStatement) {
- AssignmentStatement as = (AssignmentStatement) current;
- DataIdentifier target = as.getTarget();
- if (liveOut.containsVariable(target.getName())) {
- liveOutToTemp.put(target.getName(), Integer.valueOf(i));
- }
- }
- if (current instanceof MultiAssignmentStatement) {
- MultiAssignmentStatement mas = (MultiAssignmentStatement) current;
-
- for (DataIdentifier target : mas.getTargetList()){
- if (liveOut.containsVariable(target.getName())) {
- liveOutToTemp.put(target.getName(), Integer.valueOf(i));
- }
- }
- }
- }
-
- // only create transient read operations for variables either updated or read-before-update
- // (i.e., from LV analysis, updated and gen sets)
- if ( !liveIn.getVariables().values().isEmpty() ) {
-
- for (String varName : liveIn.getVariables().keySet()) {
-
- if (updated.containsVariable(varName) || gen.containsVariable(varName)){
-
- DataIdentifier var = liveIn.getVariables().get(varName);
- long actualDim1 = (var instanceof IndexedIdentifier) ? ((IndexedIdentifier)var).getOrigDim1() : var.getDim1();
- long actualDim2 = (var instanceof IndexedIdentifier) ? ((IndexedIdentifier)var).getOrigDim2() : var.getDim2();
- DataOp read = new DataOp(var.getName(), var.getDataType(), var.getValueType(), DataOpTypes.TRANSIENTREAD, null, actualDim1, actualDim2, var.getNnz(), var.getRowsInBlock(), var.getColumnsInBlock());
- read.setAllPositions(var.getBeginLine(), var.getBeginColumn(), var.getEndLine(), var.getEndColumn());
- ids.put(varName, read);
- }
- }
- }
-
-
- for( int i = 0; i < sb.getNumStatements(); i++ ) {
- Statement current = sb.getStatement(i);
-
- if (current instanceof OutputStatement) {
- OutputStatement os = (OutputStatement) current;
-
- DataExpression source = os.getSource();
- DataIdentifier target = os.getIdentifier();
-
- //error handling unsupported indexing expression in write statement
- if( target instanceof IndexedIdentifier ) {
- throw new LanguageException(source.printErrorLocation()+": Unsupported indexing expression in write statement. " +
- "Please, assign the right indexing result to a variable and write this variable.");
- }
-
- DataOp ae = (DataOp)processExpression(source, target, ids);
- String formatName = os.getExprParam(DataExpression.FORMAT_TYPE).toString();
- ae.setInputFormatType(Expression.convertFormatType(formatName));
-
- if (ae.getDataType() == DataType.SCALAR ) {
- ae.setOutputParams(ae.getDim1(), ae.getDim2(), ae.getNnz(), -1, -1);
- }
- else {
- switch(ae.getInputFormatType()) {
- case TEXT:
- case MM:
- case CSV:
- // write output in textcell format
- ae.setOutputParams(ae.getDim1(), ae.getDim2(), ae.getNnz(), -1, -1);
- break;
-
- case BINARY:
- // write output in binary block format
- ae.setOutputParams(ae.getDim1(), ae.getDim2(), ae.getNnz(), DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize);
- break;
-
- default:
- throw new LanguageException("Unrecognized file format: " + ae.getInputFormatType());
- }
- }
-
- output.add(ae);
-
- }
-
- if (current instanceof PrintStatement) {
- PrintStatement ps = (PrintStatement) current;
- Expression source = ps.getExpression();
- PRINTTYPE ptype = ps.getType();
-
- DataIdentifier target = createTarget();
- target.setDataType(DataType.SCALAR);
- target.setValueType(ValueType.STRING);
- target.setAllPositions(current.getFilename(), current.getBeginLine(), target.getBeginColumn(), current.getEndLine(), current.getEndColumn());
-
- Hop ae = processExpression(source, target, ids);
-
- try {
- Hop.OpOp1 op = (ptype == PRINTTYPE.PRINT ? Hop.OpOp1.PRINT : Hop.OpOp1.STOP);
- Hop printHop = new UnaryOp(target.getName(), target.getDataType(), target.getValueType(), op, ae);
- printHop.setAllPositions(current.getBeginLine(), current.getBeginColumn(), current.getEndLine(), current.getEndColumn());
- output.add(printHop);
- } catch ( HopsException e ) {
- throw new LanguageException(e);
- }
- }
-
- if (current instanceof AssignmentStatement) {
-
- AssignmentStatement as = (AssignmentStatement) current;
- DataIdentifier target = as.getTarget();
- Expression source = as.getSource();
-
-
- // CASE: regular assignment statement -- source is DML expression that is NOT user-defined or external function
- if (!(source instanceof FunctionCallIdentifier)){
-
- // CASE: target is regular data identifier
- if (!(target instanceof IndexedIdentifier)) {
-
- Hop ae = processExpression(source, target, ids);
- ids.put(target.getName(), ae);
- target.setProperties(source.getOutput());
- Integer statementId = liveOutToTemp.get(target.getName());
- if ((statementId != null) && (statementId.intValue() == i)) {
- DataOp transientwrite = new DataOp(target.getName(), target.getDataType(), target.getValueType(), ae, DataOpTypes.TRANSIENTWRITE, null);
- transientwrite.setOutputParams(ae.getDim1(), ae.getDim2(), ae.getNnz(), ae.getRowsInBlock(), ae.getColsInBlock());
- transientwrite.setAllPositions(target.getBeginLine(), target.getBeginColumn(), target.getEndLine(), target.getEndLine());
- updatedLiveOut.addVariable(target.getName(), target);
- output.add(transientwrite);
- }
- } // end if (!(target instanceof IndexedIdentifier)) {
-
- // CASE: target is indexed identifier (left-hand side indexed expression)
- else {
- Hop ae = processLeftIndexedExpression(source, (IndexedIdentifier)target, ids);
-
- ids.put(target.getName(), ae);
-
- // obtain origDim values BEFORE they are potentially updated during setProperties call
- // (this is incorrect for LHS Indexing)
- long origDim1 = ((IndexedIdentifier)target).getOrigDim1();
- long origDim2 = ((IndexedIdentifier)target).getOrigDim2();
- target.setProperties(source.getOutput());
- ((IndexedIdentifier)target).setOriginalDimensions(origDim1, origDim2);
-
- // preserve data type matrix of any index identifier
- // (required for scalar input to left indexing)
- // TODO Doug, please verify this (required for scalar assignments, otherwise TWrite of LIX target becomes scalar as well)
- if( target.getDataType() != DataType.MATRIX ) {
- target.setDataType(DataType.MATRIX);
- target.setValueType(ValueType.DOUBLE);
- target.setBlockDimensions(DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize);
- }
-
- Integer statementId = liveOutToTemp.get(target.getName());
- if ((statementId != null) && (statementId.intValue() == i)) {
- DataOp transientwrite = new DataOp(target.getName(), target.getDataType(), target.getValueType(), ae, DataOpTypes.TRANSIENTWRITE, null);
- transientwrite.setOutputParams(origDim1, origDim2, ae.getNnz(), ae.getRowsInBlock(), ae.getColsInBlock());
- transientwrite.setAllPositions(target.getBeginLine(), target.getBeginColumn(), target.getEndLine(), target.getEndColumn());
- updatedLiveOut.addVariable(target.getName(), target);
- output.add(transientwrite);
- }
- }
-
-
- }
- else
- {
- //assignment, function call
- FunctionCallIdentifier fci = (FunctionCallIdentifier) source;
- FunctionStatementBlock fsb = this._dmlProg.getFunctionStatementBlock(fci.getNamespace(),fci.getName());
-
- //error handling missing function
- if (fsb == null){
- String error = source.printErrorLocation() + "function " + fci.getName() + " is undefined in namespace " + fci.getNamespace();
- LOG.error(error);
- throw new LanguageException(error);
- }
-
- //error handling unsupported function call in indexing expression
- if( target instanceof IndexedIdentifier ) {
- String fkey = DMLProgram.constructFunctionKey(fci.getNamespace(),fci.getName());
- throw new LanguageException("Unsupported function call to '"+fkey+"' in left indexing expression. " +
- "Please, assign the function output to a variable.");
- }
-
- ArrayList<Hop> finputs = new ArrayList<Hop>();
- for (ParameterExpression paramName : fci.getParamExprs()){
- Hop in = processExpression(paramName.getExpr(), null, ids);
- finputs.add(in);
- }
-
- //create function op
- FunctionType ftype = fsb.getFunctionOpType();
- FunctionOp fcall = new FunctionOp(ftype, fci.getNamespace(), fci.getName(), finputs, new String[]{target.getName()});
- output.add(fcall);
-
- //TODO function output dataops (phase 3)
- //DataOp trFoutput = new DataOp(target.getName(), target.getDataType(), target.getValueType(), fcall, DataOpTypes.FUNCTIONOUTPUT, null);
- //DataOp twFoutput = new DataOp(target.getName(), target.getDataType(), target.getValueType(), trFoutput, DataOpTypes.TRANSIENTWRITE, null);
- }
- }
-
- else if (current instanceof MultiAssignmentStatement) {
- //multi-assignment, by definition a function call
- MultiAssignmentStatement mas = (MultiAssignmentStatement) current;
- Expression source = mas.getSource();
-
- if ( source instanceof FunctionCallIdentifier ) {
- FunctionCallIdentifier fci = (FunctionCallIdentifier) source;
- FunctionStatementBlock fsb = this._dmlProg.getFunctionStatementBlock(fci.getNamespace(),fci.getName());
- FunctionStatement fstmt = (FunctionStatement)fsb.getStatement(0);
- if (fstmt == null){
- LOG.error(source.printErrorLocation() + "function " + fci.getName() + " is undefined in namespace " + fci.getNamespace());
- throw new LanguageException(source.printErrorLocation() + "function " + fci.getName() + " is undefined in namespace " + fci.getNamespace());
- }
-
- ArrayList<Hop> finputs = new ArrayList<Hop>();
- for (ParameterExpression paramName : fci.getParamExprs()){
- Hop in = processExpression(paramName.getExpr(), null, ids);
- finputs.add(in);
- }
-
- //create function op
- String[] foutputs = new String[mas.getTargetList().size()];
- int count = 0;
- for ( DataIdentifier paramName : mas.getTargetList() ){
- foutputs[count++]=paramName.getName();
- }
-
- FunctionType ftype = fsb.getFunctionOpType();
- FunctionOp fcall = new FunctionOp(ftype, fci.getNamespace(), fci.getName(), finputs, foutputs);
- output.add(fcall);
-
- //TODO function output dataops (phase 3)
- /*for ( DataIdentifier paramName : mas.getTargetList() ){
- DataOp twFoutput = new DataOp(paramName.getName(), paramName.getDataType(), paramName.getValueType(), fcall, DataOpTypes.TRANSIENTWRITE, null);
- output.add(twFoutput);
- }*/
- }
- else if ( source instanceof BuiltinFunctionExpression && ((BuiltinFunctionExpression)source).multipleReturns() ) {
- // construct input hops
- Hop fcall = processMultipleReturnBuiltinFunctionExpression((BuiltinFunctionExpression)source, mas.getTargetList(), ids);
- output.add(fcall);
-
- }
- else
- throw new LanguageException("Class \"" + source.getClass() + "\" is not supported in Multiple Assignment statements");
- }
-
- }
- sb.updateLiveVariablesOut(updatedLiveOut);
- sb.set_hops(output);
-
- }
-
- public void constructHopsForIfControlBlock(IfStatementBlock sb) throws ParseException, LanguageException {
-
- IfStatement ifsb = (IfStatement) sb.getStatement(0);
- ArrayList<StatementBlock> ifBody = ifsb.getIfBody();
- ArrayList<StatementBlock> elseBody = ifsb.getElseBody();
-
- // construct hops for predicate in if statement
- constructHopsForConditionalPredicate(sb);
-
- // handle if statement body
- for( StatementBlock current : ifBody ) {
- constructHops(current);
- }
-
- // handle else stmt body
- for( StatementBlock current : elseBody ) {
- constructHops(current);
- }
- }
-
- /**
- * Constructs Hops for a given ForStatementBlock or ParForStatementBlock, respectively.
- *
- * @param sb
- * @throws ParseException
- * @throws LanguageException
- */
- public void constructHopsForForControlBlock(ForStatementBlock sb)
- throws ParseException, LanguageException
- {
-
- ForStatement fs = (ForStatement) sb.getStatement(0);
- ArrayList<StatementBlock> body = fs.getBody();
-
- // construct hops for iterable predicate
- constructHopsForIterablePredicate(sb);
-
- for( StatementBlock current : body ) {
- constructHops(current);
- }
- }
-
- public void constructHopsForFunctionControlBlock(FunctionStatementBlock fsb) throws ParseException, LanguageException {
-
- ArrayList<StatementBlock> body = ((FunctionStatement)fsb.getStatement(0)).getBody();
-
- for( StatementBlock current : body ) {
- constructHops(current);
- }
- }
-
- public void constructHopsForWhileControlBlock(WhileStatementBlock sb)
- throws ParseException, LanguageException {
-
- ArrayList<StatementBlock> body = ((WhileStatement)sb.getStatement(0)).getBody();
-
- // construct hops for while predicate
- constructHopsForConditionalPredicate(sb);
-
- for( StatementBlock current : body ) {
- constructHops(current);
- }
- }
-
-
- public void constructHopsForConditionalPredicate(StatementBlock passedSB) throws ParseException {
-
- HashMap<String, Hop> _ids = new HashMap<String, Hop>();
-
- // set conditional predicate
- ConditionalPredicate cp = null;
-
- if (passedSB instanceof WhileStatementBlock){
- WhileStatement ws = (WhileStatement) ((WhileStatementBlock)passedSB).getStatement(0);
- cp = ws.getConditionalPredicate();
- }
- else if (passedSB instanceof IfStatementBlock) {
- IfStatement ws = (IfStatement) ((IfStatementBlock)passedSB).getStatement(0);
- cp = ws.getConditionalPredicate();
- }
- else {
- throw new ParseException("ConditionalPredicate expected only for while or if statements.");
- }
-
- VariableSet varsRead = cp.variablesRead();
-
- for (String varName : varsRead.getVariables().keySet()) {
-
- // creating transient read for live in variables
- DataIdentifier var = passedSB.liveIn().getVariables().get(varName);
-
- DataOp read = null;
-
- if (var == null) {
- LOG.error("variable " + varName + " not live variable for conditional predicate");
- throw new ParseException("variable " + varName + " not live variable for conditional predicate");
- } else {
- long actualDim1 = (var instanceof IndexedIdentifier) ? ((IndexedIdentifier)var).getOrigDim1() : var.getDim1();
- long actualDim2 = (var instanceof IndexedIdentifier) ? ((IndexedIdentifier)var).getOrigDim2() : var.getDim2();
-
- read = new DataOp(var.getName(), var.getDataType(), var.getValueType(), DataOpTypes.TRANSIENTREAD,
- null, actualDim1, actualDim2, var.getNnz(), var.getRowsInBlock(), var.getColumnsInBlock());
- read.setAllPositions(var.getBeginLine(), var.getBeginColumn(), var.getEndLine(), var.getEndColumn());
- }
- _ids.put(varName, read);
- }
-
- DataIdentifier target = new DataIdentifier(Expression.getTempName());
- target.setDataType(DataType.SCALAR);
- target.setValueType(ValueType.BOOLEAN);
- target.setAllPositions(passedSB.getFilename(), passedSB.getBeginLine(), passedSB.getBeginColumn(), passedSB.getEndLine(), passedSB.getEndColumn());
- Hop predicateHops = null;
- Expression predicate = cp.getPredicate();
-
- if (predicate instanceof RelationalExpression) {
- predicateHops = processRelationalExpression((RelationalExpression) cp.getPredicate(), target, _ids);
- } else if (predicate instanceof BooleanExpression) {
- predicateHops = processBooleanExpression((BooleanExpression) cp.getPredicate(), target, _ids);
- } else if (predicate instanceof DataIdentifier) {
- // handle data identifier predicate
- predicateHops = processExpression(cp.getPredicate(), null, _ids);
- } else if (predicate instanceof ConstIdentifier) {
- // handle constant identifier
- // a) translate 0 --> FALSE; translate 1 --> TRUE
- // b) disallow string values
- if ( (predicate instanceof IntIdentifier && ((IntIdentifier)predicate).getValue() == 0) || (predicate instanceof DoubleIdentifier && ((DoubleIdentifier)predicate).getValue() == 0.0)) {
- cp.setPredicate(new BooleanIdentifier(false,
- predicate.getFilename(),
- predicate.getBeginLine(), predicate.getBeginColumn(),
- predicate.getEndLine(), predicate.getEndColumn()));
-
- }
- else if ( (predicate instanceof IntIdentifier && ((IntIdentifier)predicate).getValue() == 1) || (predicate instanceof DoubleIdentifier && ((DoubleIdentifier)predicate).getValue() == 1.0)) {
- cp.setPredicate(new BooleanIdentifier(true,
- predicate.getFilename(),
- predicate.getBeginLine(), predicate.getBeginColumn(),
- predicate.getEndLine(), predicate.getEndColumn()));
- }
- else if (predicate instanceof IntIdentifier || predicate instanceof DoubleIdentifier){
- cp.setPredicate(new BooleanIdentifier(true,
- predicate.getFilename(),
- predicate.getBeginLine(), predicate.getBeginColumn(),
- predicate.getEndLine(), predicate.getEndColumn()));
- LOG.warn(predicate.printWarningLocation() + "Numerical value '" + predicate.toString() + "' (!= 0/1) is converted to boolean TRUE by DML");
- }
- else if (predicate instanceof StringIdentifier) {
- LOG.error(predicate.printErrorLocation() + "String value '" + predicate.toString() + "' is not allowed for iterable predicate");
- throw new ParseException(predicate.printErrorLocation() + "String value '" + predicate.toString() + "' is not allowed for iterable predicate");
-
- }
- predicateHops = processExpression(cp.getPredicate(), null, _ids);
- }
- if (passedSB instanceof WhileStatementBlock)
- ((WhileStatementBlock)passedSB).setPredicateHops(predicateHops);
- else if (passedSB instanceof IfStatementBlock)
- ((IfStatementBlock)passedSB).setPredicateHops(predicateHops);
- }
-
-
- /**
- * Constructs all predicate Hops (for FROM, TO, INCREMENT) of an iterable predicate
- * and assigns these Hops to the passed statement block.
- *
- * Method used for both ForStatementBlock and ParForStatementBlock.
- *
- * @param passedSB
- * @throws ParseException
- */
- public void constructHopsForIterablePredicate(ForStatementBlock fsb)
- throws ParseException
- {
- HashMap<String, Hop> _ids = new HashMap<String, Hop>();
-
- // set iterable predicate
- ForStatement fs = (ForStatement) fsb.getStatement(0);
- IterablePredicate ip = fs.getIterablePredicate();
-
- for(int i=0; i < 3; i++) {
- VariableSet varsRead = null;
- if (i==0)
- varsRead = ip.getFromExpr().variablesRead();
- else if (i==1)
- varsRead = ip.getToExpr().variablesRead();
- else
- varsRead = ip.getIncrementExpr().variablesRead();
-
- if(varsRead != null) {
- for (String varName : varsRead.getVariables().keySet()) {
-
- DataIdentifier var = fsb.liveIn().getVariable(varName);
- DataOp read = null;
- if (var == null) {
- LOG.error("variable '" + varName + "' is not available for iterable predicate");
- throw new ParseException("variable '" + varName + "' is not available for iterable predicate");
- }
- else {
- long actualDim1 = (var instanceof IndexedIdentifier) ? ((IndexedIdentifier)var).getOrigDim1() : var.getDim1();
- long actualDim2 = (var instanceof IndexedIdentifier) ? ((IndexedIdentifier)var).getOrigDim2() : var.getDim2();
- read = new DataOp(var.getName(), var.getDataType(), var.getValueType(), DataOpTypes.TRANSIENTREAD,
- null, actualDim1, actualDim2, var.getNnz(), var.getRowsInBlock(), var.getColumnsInBlock());
- read.setAllPositions(var.getBeginLine(), var.getBeginColumn(), var.getEndLine(), var.getEndColumn());
- }
- _ids.put(varName, read);
- }
- }
-
- //construct hops for from, to, and increment expressions
- if(i==0)
- fsb.setFromHops( processTempIntExpression( ip.getFromExpr(), _ids ));
- else if(i==1)
- fsb.setToHops( processTempIntExpression( ip.getToExpr(), _ids ));
- else
- fsb.setIncrementHops( processTempIntExpression( ip.getIncrementExpr(), _ids ));
-
- }
-
- /*VariableSet varsRead = ip.variablesRead();
-
- for (String varName : varsRead.getVariables().keySet()) {
-
- DataIdentifier var = passedSB.liveIn().getVariable(varName);
- DataOp read = null;
- if (var == null) {
- LOG.error(var.printErrorLocation() + "variable '" + varName + "' is not available for iterable predicate");
- throw new ParseException(var.printErrorLocation() + "variable '" + varName + "' is not available for iterable predicate");
- }
- else {
- long actualDim1 = (var instanceof IndexedIdentifier) ? ((IndexedIdentifier)var).getOrigDim1() : var.getDim1();
- long actualDim2 = (var instanceof IndexedIdentifier) ? ((IndexedIdentifier)var).getOrigDim2() : var.getDim2();
- read = new DataOp(var.getName(), var.getDataType(), var.getValueType(), DataOpTypes.TRANSIENTREAD,
- null, actualDim1, actualDim2, var.getNnz(), var.getRowsInBlock(), var.getColumnsInBlock());
- read.setAllPositions(var.getBeginLine(), var.getBeginColumn(), var.getEndLine(), var.getEndColumn());
- }
- _ids.put(varName, read);
- }
-
- //construct hops for from, to, and increment expressions
- fsb.setFromHops( processTempIntExpression( ip.getFromExpr(), _ids ));
- fsb.setToHops( processTempIntExpression( ip.getToExpr(), _ids ));
- fsb.setIncrementHops( processTempIntExpression( ip.getIncrementExpr(), _ids ));*/
- }
-
-
- /**
- * Construct Hops from parse tree : Process Expression in an assignment
- * statement
- *
- * @throws ParseException
- */
- private Hop processExpression(Expression source, DataIdentifier target, HashMap<String, Hop> hops) throws ParseException {
- if (source.getKind() == Expression.Kind.BinaryOp) {
- return processBinaryExpression((BinaryExpression) source, target, hops);
- } else if (source.getKind() == Expression.Kind.RelationalOp) {
- return processRelationalExpression((RelationalExpression) source, target, hops);
- } else if (source.getKind() == Expression.Kind.BooleanOp) {
- return processBooleanExpression((BooleanExpression) source, target, hops);
- } else if (source.getKind() == Expression.Kind.Data) {
- if (source instanceof IndexedIdentifier){
- IndexedIdentifier sourceIndexed = (IndexedIdentifier) source;
- return processIndexingExpression(sourceIndexed,target,hops);
- } else if (source instanceof IntIdentifier) {
- IntIdentifier sourceInt = (IntIdentifier) source;
- LiteralOp litop = new LiteralOp(sourceInt.getValue());
- litop.setAllPositions(sourceInt.getBeginLine(), sourceInt.getBeginColumn(), sourceInt.getEndLine(), sourceInt.getEndColumn());
- setIdentifierParams(litop, sourceInt);
- return litop;
- } else if (source instanceof DoubleIdentifier) {
- DoubleIdentifier sourceDouble = (DoubleIdentifier) source;
- LiteralOp litop = new LiteralOp(sourceDouble.getValue());
- litop.setAllPositions(sourceDouble.getBeginLine(), sourceDouble.getBeginColumn(), sourceDouble.getEndLine(), sourceDouble.getEndColumn());
- setIdentifierParams(litop, sourceDouble);
- return litop;
- } else if (source instanceof DataIdentifier) {
- DataIdentifier sourceId = (DataIdentifier) source;
- return hops.get(sourceId.getName());
- } else if (source instanceof BooleanIdentifier) {
- BooleanIdentifier sourceBoolean = (BooleanIdentifier) source;
- LiteralOp litop = new LiteralOp(sourceBoolean.getValue());
- litop.setAllPositions(sourceBoolean.getBeginLine(), sourceBoolean.getBeginColumn(), sourceBoolean.getEndLine(), sourceBoolean.getEndColumn());
- setIdentifierParams(litop, sourceBoolean);
- return litop;
- } else if (source instanceof StringIdentifier) {
- StringIdentifier sourceString = (StringIdentifier) source;
- LiteralOp litop = new LiteralOp(sourceString.getValue());
- litop.setAllPositions(sourceString.getBeginLine(), sourceString.getBeginColumn(), sourceString.getEndLine(), sourceString.getEndColumn());
- setIdentifierParams(litop, sourceString);
- return litop;
- }
- } else if (source.getKind() == Expression.Kind.BuiltinFunctionOp) {
- try {
- return processBuiltinFunctionExpression((BuiltinFunctionExpression) source, target, hops);
- } catch (HopsException e) {
- throw new ParseException(e.getMessage());
- }
- } else if (source.getKind() == Expression.Kind.ParameterizedBuiltinFunctionOp ) {
- try {
- return processParameterizedBuiltinFunctionExpression((ParameterizedBuiltinFunctionExpression)source, target, hops);
- } catch ( HopsException e ) {
- throw new ParseException(e.getMessage());
- }
- } else if (source.getKind() == Expression.Kind.DataOp ) {
- try {
- Hop ae = (Hop)processDataExpression((DataExpression)source, target, hops);
-
- if (ae instanceof DataOp){
- String formatName = ((DataExpression)source).getVarParam(DataExpression.FORMAT_TYPE).toString();
- ((DataOp)ae).setInputFormatType(Expression.convertFormatType(formatName));
- }
- //hops.put(target.getName(), ae);
- return ae;
- } catch ( Exception e ) {
- throw new ParseException(e.getMessage());
- }
- }
- return null;
- } // end method processExpression
-
- private DataIdentifier createTarget(Expression source) {
- Identifier id = source.getOutput();
- if (id instanceof DataIdentifier && !(id instanceof DataExpression))
- return (DataIdentifier) id;
- DataIdentifier target = new DataIdentifier(Expression.getTempName());
- target.setProperties(id);
- return target;
- }
-
- private DataIdentifier createTarget() {
- DataIdentifier target = new DataIdentifier(Expression.getTempName());
- return target;
- }
-
-
- /**
- * Constructs the Hops for arbitrary expressions that eventually evaluate to an INT scalar.
- *
- * @param source
- * @param hops
- * @return
- * @throws ParseException
- */
- private Hop processTempIntExpression( Expression source, HashMap<String, Hop> hops )
- throws ParseException
- {
- DataIdentifier tmpOut = createTarget();
- tmpOut.setDataType(DataType.SCALAR);
- tmpOut.setValueType(ValueType.INT);
- source.setOutput(tmpOut);
-
- return processExpression(source, tmpOut, hops );
- }
-
- private Hop processLeftIndexedExpression(Expression source, IndexedIdentifier target, HashMap<String, Hop> hops)
- throws ParseException {
-
- // process target indexed expressions
- Hop rowLowerHops = null, rowUpperHops = null, colLowerHops = null, colUpperHops = null;
-
- if (target.getRowLowerBound() != null)
- rowLowerHops = processExpression(target.getRowLowerBound(),null,hops);
- else
- rowLowerHops = new LiteralOp(1);
-
- if (target.getRowUpperBound() != null)
- rowUpperHops = processExpression(target.getRowUpperBound(),null,hops);
- else
- {
- if ( target.getDim1() != -1 )
- rowUpperHops = new LiteralOp(target.getOrigDim1());
- else
- {
- try {
- //currBuiltinOp = new UnaryOp(target.getName(), target.getDataType(), target.getValueType(), Hops.OpOp1.NROW, expr);
- rowUpperHops = new UnaryOp(target.getName(), DataType.SCALAR, ValueType.INT, Hop.OpOp1.NROW, hops.get(target.getName()));
- rowUpperHops.setAllPositions(target.getBeginLine(), target.getBeginColumn(), target.getEndLine(), target.getEndColumn());
- } catch (HopsException e) {
- LOG.error(target.printErrorLocation() + "error processing row upper index for indexed expression " + target.toString());
- throw new RuntimeException(target.printErrorLocation() + "error processing row upper index for indexed expression " + target.toString());
- }
- }
- }
- if (target.getColLowerBound() != null)
- colLowerHops = processExpression(target.getColLowerBound(),null,hops);
- else
- colLowerHops = new LiteralOp(1);
-
- if (target.getColUpperBound() != null)
- colUpperHops = processExpression(target.getColUpperBound(),null,hops);
- else
- {
- if ( target.getDim2() != -1 )
- colUpperHops = new LiteralOp(target.getOrigDim2());
- else
- {
- try {
- colUpperHops = new UnaryOp(target.getName(), DataType.SCALAR, ValueType.INT, Hop.OpOp1.NCOL, hops.get(target.getName()));
- } catch (HopsException e) {
- LOG.error(target.printErrorLocation() + " error processing column upper index for indexed expression " + target.toString());
- throw new RuntimeException(target.printErrorLocation() + " error processing column upper index for indexed expression " + target.toString(), e);
- }
- }
- }
-
- //if (target == null) {
- // target = createTarget(source);
- //}
-
- // process the source expression to get source Hops
- Hop sourceOp = processExpression(source, target, hops);
-
- // process the target to get targetHops
- Hop targetOp = hops.get(target.getName());
- if (targetOp == null){
- LOG.error(target.printErrorLocation() + " must define matrix " + target.getName() + " before indexing operations are allowed ");
- throw new ParseException(target.printErrorLocation() + " must define matrix " + target.getName() + " before indexing operations are allowed ");
- }
-
- //TODO Doug, please verify this (we need probably a cleaner way than this postprocessing)
- if( sourceOp.getDataType() == DataType.MATRIX && source.getOutput().getDataType() == DataType.SCALAR )
- sourceOp.setDataType(DataType.SCALAR);
-
- Hop leftIndexOp = new LeftIndexingOp(target.getName(), target.getDataType(), target.getValueType(),
- targetOp, sourceOp, rowLowerHops, rowUpperHops, colLowerHops, colUpperHops,
- target.getRowLowerEqualsUpper(), target.getColLowerEqualsUpper());
-
- setIdentifierParams(leftIndexOp, target);
-
- leftIndexOp.setAllPositions(target.getBeginLine(), target.getBeginColumn(), target.getEndLine(), target.getEndColumn());
- leftIndexOp.setDim1(target.getOrigDim1());
- leftIndexOp.setDim2(target.getOrigDim2());
-
- return leftIndexOp;
- }
-
-
- private Hop processIndexingExpression(IndexedIdentifier source, DataIdentifier target, HashMap<String, Hop> hops)
- throws ParseException {
-
- // process Hops for indexes (for source)
- Hop rowLowerHops = null, rowUpperHops = null, colLowerHops = null, colUpperHops = null;
-
- if (source.getRowLowerBound() != null)
- rowLowerHops = processExpression(source.getRowLowerBound(),null,hops);
- else
- rowLowerHops = new LiteralOp(1);
-
- if (source.getRowUpperBound() != null)
- rowUpperHops = processExpression(source.getRowUpperBound(),null,hops);
- else
- {
- if ( source.getOrigDim1() != -1 )
- rowUpperHops = new LiteralOp(source.getOrigDim1());
- else
- {
- try {
- //currBuiltinOp = new UnaryOp(target.getName(), target.getDataType(), target.getValueType(), Hops.OpOp1.NROW, expr);
- rowUpperHops = new UnaryOp(source.getName(), DataType.SCALAR, ValueType.INT, Hop.OpOp1.NROW, hops.get(source.getName()));
- rowUpperHops.setAllPositions(source.getBeginLine(),source.getBeginColumn(), source.getEndLine(), source.getEndColumn());
- } catch (HopsException e) {
- LOG.error(source.printErrorLocation() + "error processing row upper index for indexed identifier " + source.toString());
- throw new RuntimeException(source.printErrorLocation() + "error processing row upper index for indexed identifier " + source.toString() + e);
- }
- }
- }
- if (source.getColLowerBound() != null)
- colLowerHops = processExpression(source.getColLowerBound(),null,hops);
- else
- colLowerHops = new LiteralOp(1);
-
- if (source.getColUpperBound() != null)
- colUpperHops = processExpression(source.getColUpperBound(),null,hops);
- else
- {
- if ( source.getOrigDim2() != -1 )
- colUpperHops = new LiteralOp(source.getOrigDim2());
- else
- {
- try {
- colUpperHops = new UnaryOp(source.getName(), DataType.SCALAR, ValueType.INT, Hop.OpOp1.NCOL, hops.get(source.getName()));
- } catch (HopsException e) {
- LOG.error(source.printErrorLocation() + "error processing column upper index for indexed indentifier " + source.toString(), e);
- throw new RuntimeException(source.printErrorLocation() + "error processing column upper index for indexed indentifier " + source.toString(), e);
- }
- }
- }
-
- if (target == null) {
- target = createTarget(source);
- }
- //unknown nnz after range indexing (applies to indexing op but also
- //data dependent operations)
- target.setNnz(-1);
-
- Hop indexOp = new IndexingOp(target.getName(), target.getDataType(), target.getValueType(),
- hops.get(source.getName()), rowLowerHops, rowUpperHops, colLowerHops, colUpperHops,
- source.getRowLowerEqualsUpper(), source.getColLowerEqualsUpper());
-
- indexOp.setAllPositions(indexOp.getBeginLine(), indexOp.getBeginColumn(), indexOp.getEndLine(), indexOp.getEndColumn());
- setIdentifierParams(indexOp, target);
-
- return indexOp;
- }
-
-
- /**
- * Construct Hops from parse tree : Process Binary Expression in an
- * assignment statement
- *
- * @throws ParseException
- */
- private Hop processBinaryExpression(BinaryExpression source, DataIdentifier target, HashMap<String, Hop> hops)
- throws ParseException
- {
- Hop left = processExpression(source.getLeft(), null, hops);
- Hop right = processExpression(source.getRight(), null, hops);
-
- if (left == null || right == null){
- left = processExpression(source.getLeft(), null, hops);
- right = processExpression(source.getRight(), null, hops);
- }
-
- Hop currBop = null;
-
- //prepare target identifier and ensure that output type is of inferred type
- //(type should not be determined by target (e.g., string for print)
- if (target == null) {
- target = createTarget(source);
- }
- target.setValueType(source.getOutput().getValueType());
-
- if (source.getOpCode() == Expression.BinaryOp.PLUS) {
- currBop = new BinaryOp(target.getName(), target.getDataType(), target.getValueType(), OpOp2.PLUS, left, right);
- } else if (source.getOpCode() == Expression.BinaryOp.MINUS) {
- currBop = new BinaryOp(target.getName(), target.getDataType(), target.getValueType(), OpOp2.MINUS, left, right);
- } else if (source.getOpCode() == Expression.BinaryOp.MULT) {
- currBop = new BinaryOp(target.getName(), target.getDataType(), target.getValueType(), OpOp2.MULT, left, right);
- } else if (source.getOpCode() == Expression.BinaryOp.DIV) {
- currBop = new BinaryOp(target.getName(), target.getDataType(), target.getValueType(), OpOp2.DIV, left, right);
- } else if (source.getOpCode() == Expression.BinaryOp.MODULUS) {
- currBop = new BinaryOp(target.getName(), target.getDataType(), target.getValueType(), OpOp2.MODULUS, left, right);
- } else if (source.getOpCode() == Expression.BinaryOp.INTDIV) {
- currBop = new BinaryOp(target.getName(), target.getDataType(), target.getValueType(), OpOp2.INTDIV, left, right);
- } else if (source.getOpCode() == Expression.BinaryOp.MATMULT) {
- currBop = new AggBinaryOp(target.getName(), target.getDataType(), target.getValueType(), OpOp2.MULT, AggOp.SUM, left, right);
- } else if (source.getOpCode() == Expression.BinaryOp.POW) {
- currBop = new BinaryOp(target.getName(), target.getDataType(), target.getValueType(), OpOp2.POW, left, right);
- }
- else {
- throw new ParseException("Unsupported parsing of binary expression: "+source.getOpCode());
- }
- setIdentifierParams(currBop, source.getOutput());
- currBop.setAllPositions(source.getBeginLine(), source.getBeginColumn(), source.getEndLine(), source.getEndColumn());
- return currBop;
-
- }
-
- private Hop processRelationalExpression(RelationalExpression source, DataIdentifier target,
- HashMap<String, Hop> hops) throws ParseException {
-
- Hop left = processExpression(source.getLeft(), null, hops);
- Hop right = processExpression(source.getRight(), null, hops);
-
- Hop currBop = null;
-
- if (target == null) {
- target = createTarget(source);
- if(left.getDataType() == DataType.MATRIX || right.getDataType() == DataType.MATRIX) {
- // Added to support matrix relational comparison
- target.setDataType(DataType.MATRIX);
- target.setValueType(ValueType.BOOLEAN);
- }
- else {
- // Added to support scalar relational comparison
- target.setDataType(DataType.SCALAR);
- target.setValueType(ValueType.BOOLEAN);
- }
- }
-
- OpOp2 op = null;
-
- if (source.getOpCode() == Expression.RelationalOp.LESS) {
- op = OpOp2.LESS;
- } else if (source.getOpCode() == Expression.RelationalOp.LESSEQUAL) {
- op = OpOp2.LESSEQUAL;
- } else if (source.getOpCode() == Expression.RelationalOp.GREATER) {
- op = OpOp2.GREATER;
- } else if (source.getOpCode() == Expression.RelationalOp.GREATEREQUAL) {
- op = OpOp2.GREATEREQUAL;
- } else if (source.getOpCode() == Expression.RelationalOp.EQUAL) {
- op = OpOp2.EQUAL;
- } else if (source.getOpCode() == Expression.RelationalOp.NOTEQUAL) {
- op = OpOp2.NOTEQUAL;
- }
- currBop = new BinaryOp(target.getName(), target.getDataType(), target.getValueType(), op, left, right);
- currBop.setAllPositions(source.getBeginLine(), source.getBeginColumn(), source.getEndLine(), source.getEndColumn());
- return currBop;
- }
-
- /**
- *
- * @param source
- * @param target
- * @param hops
- * @return
- * @throws ParseException
- */
- private Hop processBooleanExpression(BooleanExpression source, DataIdentifier target, HashMap<String, Hop> hops)
- throws ParseException
- {
- // Boolean Not has a single parameter
- boolean constLeft = (source.getLeft().getOutput() instanceof ConstIdentifier);
- boolean constRight = false;
- if (source.getRight() != null) {
- constRight = (source.getRight().getOutput() instanceof ConstIdentifier);
- }
-
- if (constLeft || constRight) {
- LOG.error(source.printErrorLocation() + "Boolean expression with constant unsupported");
- throw new RuntimeException(source.printErrorLocation() + "Boolean expression with constant unsupported");
- }
-
- Hop left = processExpression(source.getLeft(), null, hops);
- Hop right = null;
- if (source.getRight() != null) {
- right = processExpression(source.getRight(), null, hops);
- }
-
- //prepare target identifier and ensure that output type is boolean
- //(type should not be determined by target (e.g., string for print)
- if (target == null) {
- target = createTarget(source);
- }
- target.setValueType(ValueType.BOOLEAN);
-
- if (source.getRight() == null) {
- Hop currUop = null;
- try {
- currUop = new UnaryOp(target.getName(), target.getDataType(), target.getValueType(), Hop.OpOp1.NOT, left);
- currUop.setAllPositions(source.getBeginLine(), source.getBeginColumn(), source.getEndLine(), source.getEndColumn());
- } catch (HopsException e) {
- throw new ParseException(e.getMessage());
- }
- return currUop;
- } else {
- Hop currBop = null;
- OpOp2 op = null;
-
- if (source.getOpCode() == Expression.BooleanOp.LOGICALAND) {
- op = OpOp2.AND;
- } else if (source.getOpCode() == Expression.BooleanOp.LOGICALOR) {
- op = OpOp2.OR;
- } else {
- LOG.error(source.printErrorLocation() + "Unknown boolean operation " + source.getOpCode());
- throw new RuntimeException(source.printErrorLocation() + "Unknown boolean operation " + source.getOpCode());
- }
- currBop = new BinaryOp(target.getName(), target.getDataType(), target.getValueType(), op, left, right);
- currBop.setAllPositions(source.getBeginLine(), source.getBeginColumn(), source.getEndLine(), source.getEndColumn());
- // setIdentifierParams(currBop,source.getOutput());
- return currBop;
- }
- }
-
- private Hop constructDfHop(String name, DataType dt, ValueType vt, ParameterizedBuiltinFunctionOp op, HashMap<String,Hop> paramHops) throws HopsException {
-
- // Add a hop to paramHops to store distribution information.
- // Distribution parameter hops would have been already present in paramHops.
- Hop distLop = null;
- switch(op) {
- case QNORM:
- case PNORM:
- distLop = new LiteralOp("normal");
- break;
- case QT:
- case PT:
- distLop = new LiteralOp("t");
- break;
- case QF:
- case PF:
- distLop = new LiteralOp("f");
- break;
- case QCHISQ:
- case PCHISQ:
- distLop = new LiteralOp("chisq");
- break;
- case QEXP:
- case PEXP:
- distLop = new LiteralOp("exp");
- break;
-
- case CDF:
- case INVCDF:
- break;
-
- default:
- throw new HopsException("Invalid operation: " + op);
- }
- if (distLop != null)
- paramHops.put("dist", distLop);
-
- return new ParameterizedBuiltinOp(name, dt, vt, ParameterizedBuiltinFunctionExpression.pbHopMap.get(op), paramHops);
- }
-
- /**
- * Construct Hops from parse tree : Process ParameterizedBuiltinFunction Expression in an
- * assignment statement
- *
- * @throws ParseException
- * @throws HopsException
- */
- private Hop processParameterizedBuiltinFunctionExpression(ParameterizedBuiltinFunctionExpression source, DataIdentifier target,
- HashMap<String, Hop> hops) throws ParseException, HopsException {
-
- // this expression has multiple "named" parameters
- HashMap<String, Hop> paramHops = new HashMap<String,Hop>();
-
- // -- construct hops for all input parameters
- // -- store them in hashmap so that their "name"s are maintained
- Hop pHop = null;
- for ( String paramName : source.getVarParams().keySet() ) {
- pHop = processExpression(source.getVarParam(paramName), null, hops);
- paramHops.put(paramName, pHop);
- }
-
- Hop currBuiltinOp = null;
-
- if (target == null) {
- target = createTarget(source);
- }
-
- // construct hop based on opcode
- switch(source.getOpCode()) {
- case CDF:
- case INVCDF:
- case QNORM:
- case QT:
- case QF:
- case QCHISQ:
- case QEXP:
- case PNORM:
- case PT:
- case PF:
- case PCHISQ:
- case PEXP:
- currBuiltinOp = constructDfHop(target.getName(), target.getDataType(), target.getValueType(), source.getOpCode(), paramHops);
- break;
-
- case GROUPEDAGG:
- currBuiltinOp = new ParameterizedBuiltinOp(
- target.getName(), target.getDataType(), target.getValueType(), ParamBuiltinOp.GROUPEDAGG, paramHops);
- break;
-
- case RMEMPTY:
- currBuiltinOp = new ParameterizedBuiltinOp(
- target.getName(), target.getDataType(), target.getValueType(), ParamBuiltinOp.RMEMPTY, paramHops);
- break;
-
- case REPLACE:
- currBuiltinOp = new ParameterizedBuiltinOp(
- target.getName(), target.getDataType(), target.getValueType(), ParamBuiltinOp.REPLACE, paramHops);
- break;
-
- case ORDER:
- ArrayList<Hop> inputs = new ArrayList<Hop>();
- inputs.add(paramHops.get("target"));
- inputs.add(paramHops.get("by"));
- inputs.add(paramHops.get("decreasing"));
- inputs.add(paramHops.get("index.return"));
-
- currBuiltinOp = new ReorgOp(target.getName(), target.getDataType(), target.getValueType(), ReOrgOp.SORT, inputs);
-
- break;
-
- case TRANSFORM:
- currBuiltinOp = new ParameterizedBuiltinOp(
- target.getName(), target.getDataType(),
- target.getValueType(), ParamBuiltinOp.TRANSFORM,
- paramHops);
- break;
-
- default:
-
- LOG.error(source.printErrorLocation() +
- "processParameterizedBuiltinFunctionExpression() -- Unknown operation: "
- + source.getOpCode());
-
- throw new ParseException(source.printErrorLocation() +
- "processParameterizedBuiltinFunctionExpression() -- Unknown operation: "
- + source.getOpCode());
- }
-
- setIdentifierParams(currBuiltinOp, source.getOutput());
-
- currBuiltinOp.setAllPositions(source.getBeginLine(), source.getBeginColumn(), source.getEndLine(), source.getEndColumn());
-
- return currBuiltinOp;
- }
-
- /**
- * Construct Hops from parse tree : Process ParameterizedExpression in a
- * read/write/rand statement
- *
- * @throws ParseException
- * @throws HopsException
- */
- private Hop processDataExpression(DataExpression source, DataIdentifier target,
- HashMap<String, Hop> hops) throws ParseException, HopsException {
-
- // this expression has multiple "named" parameters
- HashMap<String, Hop> paramHops = new HashMap<String,Hop>();
-
- // -- construct hops for all input parameters
- // -- store them in hashmap so that their "name"s are maintained
- Hop pHop = null;
- for ( String paramName : source.getVarParams().keySet() ) {
- pHop = processExpression(source.getVarParam(paramName), null, hops);
- paramHops.put(paramName, pHop);
- }
-
- Hop currBuiltinOp = null;
-
- if (target == null) {
- target = createTarget(source);
- }
-
- // construct hop based on opcode
- switch(source.getOpCode()) {
- case READ:
- currBuiltinOp = new DataOp(target.getName(), target.getDataType(), target.getValueType(), DataOpTypes.PERSISTENTREAD, paramHops);
- ((DataOp)currBuiltinOp).setFileName(((StringIdentifier)source.getVarParam(DataExpression.IO_FILENAME)).getValue());
- break;
-
- case WRITE:
- String name = target.getName();
- currBuiltinOp = new DataOp(
- target.getName(), target.getDataType(), target.getValueType(), DataOpTypes.PERSISTENTWRITE, hops.get(name), paramHops);
- //MB: commented for dynamic write
- /*Identifier ioFilename = (Identifier)source.getVarParam(DataExpression.IO_FILENAME);
- if (!(ioFilename instanceof StringIdentifier)) {
- LOG.error(source.printErrorLocation() + "processDataExpression():: Filename must be a constant string value");
- throw new ParseException(source.printErrorLocation() + "processDataExpression():: Filename must be a constant string value");
- } else {
- ((DataOp)currBuiltinOp).setFileName(((StringIdentifier)ioFilename).getValue());
- }*/
- break;
-
- case RAND:
- // We limit RAND_MIN, RAND_MAX, RAND_SPARSITY, RAND_SEED, and RAND_PDF to be constants
- DataGenMethod method = (paramHops.get(DataExpression.RAND_MIN).getValueType()==ValueType.STRING) ?
- DataGenMethod.SINIT : DataGenMethod.RAND;
- currBuiltinOp = new DataGenOp(method, target, paramHops);
- break;
-
- case MATRIX:
- ArrayList<Hop> tmp = new ArrayList<Hop>();
- tmp.add( 0, paramHops.get(DataExpression.RAND_DATA) );
- tmp.add( 1, paramHops.get(DataExpression.RAND_ROWS) );
- tmp.add( 2, paramHops.get(DataExpression.RAND_COLS) );
- tmp.add( 3, paramHops.get(DataExpression.RAND_BY_ROW) );
- currBuiltinOp = new ReorgOp(target.getName(), target.getDataType(), target.getValueType(), ReOrgOp.RESHAPE, tmp);
- break;
-
-
- default:
- LOG.error(source.printErrorLocation() +
- "processDataExpression():: Unknown operation: "
- + source.getOpCode());
-
- throw new ParseException(source.printErrorLocation() +
- "processDataExpression():: Unknown operation: "
- + source.getOpCode());
- }
-
- //set identifier meta data (incl dimensions and blocksizes)
- setIdentifierParams(currBuiltinOp, source.getOutput());
- if( source.getOpCode()==DataExpression.DataOp.READ )
- ((DataOp)currBuiltinOp).setInputBlockSizes(target.getRowsInBlock(), target.getColumnsInBlock());
- currBuiltinOp.setAllPositions(source.getBeginLine(), source.getBeginColumn(), source.getEndLine(), source.getEndColumn());
-
- return currBuiltinOp;
- }
-
- /**
- * Construct HOps from parse tree: process BuiltinFunction Expressions in
- * MultiAssignment Statements. For all other builtin function expressions,
- * <code>processBuiltinFunctionExpression()</code> is used.
- */
- private Hop processMultipleReturnBuiltinFunctionExpression(BuiltinFunctionExpression source, ArrayList<DataIdentifier> targetList,
- HashMap<String, Hop> hops) throws ParseException {
-
- // Construct Hops for all inputs
- ArrayList<Hop> inputs = new ArrayList<Hop>();
- inputs.add( processExpression(source.getFirstExpr(), null, hops) );
- if ( source.getSecondExpr() != null )
- inputs.add( processExpression(source.getSecondExpr(), null, hops) );
- if ( source.getThirdExpr() != null )
- inputs.add( processExpression(source.getThirdExpr(), null, hops) );
-
- FunctionType ftype = FunctionType.MULTIRETURN_BUILTIN;
- String nameSpace = DMLProgram.INTERNAL_NAMESPACE;
-
- // Create an array list to hold the outputs of this lop.
- // Exact list of outputs are added based on opcode.
- ArrayList<Hop> outputs = new ArrayList<Hop>();
-
- // Construct Hop for current builtin function expression based on its type
- Hop currBuiltinOp = null;
- switch (source.getOpCode()) {
- case QR:
- case LU:
- case EIGEN:
-
- // Number of outputs = size of targetList = #of identifiers in source.getOutputs
- String[] outputNames = new String[targetList.size()];
- for ( int i=0; i < targetList.size(); i++ ) {
- outputNames[i] = ((DataIdentifier)targetList.get(i)).getName();
- Hop output = new DataOp(outputNames[i], DataType.MATRIX, ValueType.DOUBLE, inputs.get(0), DataOpTypes.FUNCTIONOUTPUT, outputNames[i]);
- outputs.add(output);
- }
-
- // Create the hop for current function call
- FunctionOp fcall = new FunctionOp(ftype, nameSpace, source.getOpCode().toString(), inputs, outputNames, outputs);
- currBuiltinOp = fcall;
-
- break;
-
- default:
- throw new ParseException("Invaid Opcode in DMLTranslator:processMultipleReturnBuiltinFunctionExpression(): " + source.getOpCode());
- }
-
- // set properties for created hops based on outputs of source expression
- for ( int i=0; i < source.getOutputs().length; i++ ) {
- setIdentifierParams( outputs.get(i), source.getOutputs()[i]);
- outputs.get(i).setAllPositions(source.getBeginLine(), source.getBeginColumn(), source.getEndLine(), source.getEndColumn());
- }
- currBuiltinOp.setAllPositions(source.getBeginLine(), source.getBeginColumn(), source.getEndLine(), source.getEndColumn());
-
- return currBuiltinOp;
- }
-
- /**
- * Construct Hops from parse tree : Process BuiltinFunction Expression in an
- * assignment statement
- *
- * @throws ParseException
- * @throws HopsException
- */
- private Hop processBuiltinFunctionExpression(BuiltinFunctionExpression source, DataIdentifier target,
- HashMap<String, Hop> hops) throws ParseException, HopsException {
- Hop expr = processExpression(source.getFirstExpr(), null, hops);
- Hop expr2 = null;
- if (source.getSecondExpr() != null) {
- expr2 = processExpression(source.getSecondExpr(), null, hops);
- }
- Hop expr3 = null;
- if (source.getThirdExpr() != null) {
- expr3 = processExpression(source.getThirdExpr(), null, hops);
- }
-
- Hop currBuiltinOp = null;
-
- if (target == null) {
- target = createTarget(source);
- }
-
- // Construct the hop based on the type of Builtin function
- switch (source.getOpCode()) {
-
- case COLSUM:
- currBuiltinOp = new AggUnaryOp(target.getName(), target.getDataType(), target.getValueType(), AggOp.SUM,
- Direction.Col, expr);
- break;
-
- case COLMAX:
- currBuiltinOp = new AggUnaryOp(target.getName(), target.getDataType(), target.getValueType(), AggOp.MAX,
- Direction.Col, expr);
- break;
-
- case COLMIN:
- currBuiltinOp = new AggUnaryOp(target.getName(), target.getDataType(), target.getValueType(), AggOp.MIN,
- Direction.Col, expr);
- break;
-
- case COLMEAN:
- // hop to compute colSums
- currBuiltinOp = new AggUnaryOp(target.getName(), target.getDataType(), target.getValueType(), AggOp.MEAN,
- Direction.Col, expr);
- break;
-
- case ROWSUM:
- currBuiltinOp = new AggUnaryOp(target.getName(), target.getDataType(), target.getValueType(), AggOp.SUM,
- Direction.Row, expr);
- break;
-
- case ROWMAX:
- currBuiltinOp = new AggUnaryOp(target.getName(), target.getDataType(), target.getValueType(), AggOp.MAX,
- Direction.Row, expr);
- break;
-
- case ROWINDEXMAX:
- currBuiltinOp = new AggUnaryOp(target.getName(), target.getDataType(), target.getValueType(), AggOp.MAXINDEX,
- Direction.Row, expr);
- break;
-
- case ROWINDEXMIN:
- currBuiltinOp = new AggUnaryOp(target.getName(), target.getDataType(), target.getValueType(), AggOp.MININDEX,
- Direction.Row, expr);
- break;
-
- case ROWMIN:
- currBuiltinOp = new AggUnaryOp(target.getName(), target.getDataType(), target.getValueType(), AggOp.MIN,
- Direction.Row, expr);
- break;
-
- case ROWMEAN:
- currBuiltinOp = new AggUnaryOp(target.getName(), target.getDataType(), target.getValueType(), AggOp.MEAN,
- Direction.Row, expr);
- break;
-
- case NROW:
- // If the dimensions are available at compile time, then create a LiteralOp (constant propagation)
- // Else create a UnaryOp so that a control program instruction is generated
-
- long nRows = expr.getDim1();
- if (nRows == -1) {
- currBuiltinOp = new UnaryOp(target.getName(), target.getDataType(), target.getValueType(), Hop.OpOp1.NROW, expr);
- }
- else {
- currBuiltinOp = new LiteralOp(nRows);
- }
- break;
-
- case NCOL:
- // If the dimensions are available at compile time, then create a LiteralOp (constant propagation)
- // Else create a UnaryOp so that a control program instruction is generated
-
- long nCols = expr.getDim2();
- if (nCols == -1) {
- currBuiltinOp = new UnaryOp(target.getName(), target.getDataType(), target.getValueType(), Hop.OpOp1.NCOL, expr);
- }
- else {
- currBuiltinOp = new LiteralOp(nCols);
- }
- break;
- case LENGTH:
- long nRows2 = expr.getDim1();
- long nCols2 = expr.getDim2();
- /*
- * If the dimensions are available at compile time, then create a LiteralOp (constant propagation)
- * Else create a UnaryOp so that a control program instruction is generated
- */
- if ((nCols2 == -1) || (nRows2 == -1)) {
- currBuiltinOp = new UnaryOp(target.getName(), target.getDataType(), target.getValueType(), Hop.OpOp1.LENGTH, expr);
- }
- else {
- long lval = (nCols2 * nRows2);
- currBuiltinOp = new LiteralOp(lval);
- }
- break;
-
- case SUM:
- currBuiltinOp = new AggUnaryOp(target.getName(), target.getDataType(), target.getValueType(), AggOp.SUM,
- Direction.RowCol, expr);
- break;
-
- case MEAN:
- if ( expr2 == null ) {
- // example: x = mean(Y);
- currBuiltinOp = new AggUnaryOp(target.getName(), target.getDataType(), target.getValueType(), AggOp.MEAN,
- Direction.RowCol, expr);
- }
- else {
- // example: x = mean(Y,W);
- // stable weighted mean is implemented by using centralMoment with order = 0
- Hop orderHop = new LiteralOp(0);
- currBuiltinOp=new TernaryOp(target.getName(), target.getDataType(), target.getValueType(),
- Hop.OpOp3.CENTRALMOMENT, expr, expr2, orderHop);
- }
- break;
-
- case MIN:
- //construct AggUnary for min(X) but BinaryOp for min(X,Y)
- if( expr2 == null ) {
- currBuiltinOp = new AggUnaryOp(target.getName(), target.getDataType(), target.getValueType(),
- AggOp.MIN, Direction.RowCol, expr);
- }
- else {
- currBuiltinOp = new BinaryOp(target.getName(), target.getDataType(), target.getValueType(), OpOp2.MIN,
- expr, expr2);
- }
- break;
- case MAX:
- //construct AggUnary for max(X) but BinaryOp for max(X,Y)
- if( expr2 == null ) {
- currBuiltinOp = new AggUnaryOp(target.getName(), target.getDataType(), target.getValueType(),
- AggOp.MAX, Direction.RowCol, expr);
- } else {
- currBuiltinOp = new BinaryOp(target.getName(), target.getDataType(), target.getValueType(), OpOp2.MAX,
- expr, expr2);
- }
- break;
-
- case PPRED:
- String sop = ((StringIdentifier)source.getThirdExpr()).getValue();
- sop = sop.replace("\"", "");
- OpOp2 operation;
- if ( sop.equalsIgnoreCase(">=") )
- operation = OpOp2.GREATEREQUAL;
- else if ( sop.equalsIgnoreCase(">") )
- operation =
<TRUNCATED>