You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by lr...@apache.org on 2015/12/03 19:45:42 UTC
[11/78] [abbrv] [partial] incubator-systemml git commit: Move files
to new package folder structure
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/parser/python/PydmlSyntacticValidatorHelper.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/parser/python/PydmlSyntacticValidatorHelper.java b/src/main/java/com/ibm/bi/dml/parser/python/PydmlSyntacticValidatorHelper.java
deleted file mode 100644
index 2805d9c..0000000
--- a/src/main/java/com/ibm/bi/dml/parser/python/PydmlSyntacticValidatorHelper.java
+++ /dev/null
@@ -1,123 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.parser.python;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.antlr.v4.runtime.Token;
-
-import com.ibm.bi.dml.parser.DMLProgram;
-import com.ibm.bi.dml.parser.python.PydmlParser.FunctionCallAssignmentStatementContext;
-import com.ibm.bi.dml.parser.python.PydmlParser.ParameterizedExpressionContext;
-import com.ibm.bi.dml.parser.python.PydmlSyntacticErrorListener.CustomDmlErrorListener;
-
-
-public class PydmlSyntacticValidatorHelper {
-
- private CustomDmlErrorListener _errorListener = null;
- public PydmlSyntacticValidatorHelper(CustomDmlErrorListener errorListener) {
- this._errorListener = errorListener;
- }
- public void notifyErrorListeners(String message, int line, int charPositionInLine) {
- this._errorListener.validationError(line, charPositionInLine, message);
- }
-
- public void notifyErrorListeners(String message, Token op) {
- this._errorListener.validationError(op.getLine(), op.getCharPositionInLine(), message);
- }
-
- public String getCurrentFileName() {
- return _errorListener.peekFileName();
- }
-
- // Returns list of two elements <namespace, function names>, else null
- public ArrayList<String> getQualifiedNames(String fullyQualifiedFunctionName) {
- String [] fnNames = fullyQualifiedFunctionName.split("\\."); // instead of ::
- String functionName = "";
- String namespace = "";
- if(fnNames.length == 1) {
- namespace = DMLProgram.DEFAULT_NAMESPACE;
- functionName = fnNames[0].trim();
- }
- else if(fnNames.length == 2) {
- namespace = fnNames[0].trim();
- functionName = fnNames[1].trim();
- }
- else
- return null;
-
- ArrayList<String> retVal = new ArrayList<String>();
- retVal.add(namespace);
- retVal.add(functionName);
- return retVal;
- }
-
-// public static void setInfoForArithmeticOp(com.ibm.bi.dml.parser.Expression current,
-// com.ibm.bi.dml.parser.Expression left,
-// com.ibm.bi.dml.parser.Expression right, String opStr) {
-// try {
-// // PLUS, MINUS, MULT, DIV, MODULUS, INTDIV, MATMULT, POW, INVALID
-// com.ibm.bi.dml.parser.Expression.BinaryOp bop = com.ibm.bi.dml.parser.Expression.getBinaryOp(opStr);
-// current = new com.ibm.bi.dml.parser.BinaryExpression(bop);
-// ((com.ibm.bi.dml.parser.BinaryExpression)current).setLeft(left);
-// ((com.ibm.bi.dml.parser.BinaryExpression)current).setRight(right);
-// ((com.ibm.bi.dml.parser.BinaryExpression)current).setFilename(DmlSyntacticErrorListener.currentFileName.peek());
-// }
-// catch(Exception e) {
-// System.out.println("In setInfoForArithmeticOp>>");
-// e.printStackTrace();
-// }
-// }
-
-// public static void setInfoForBooleanOp(com.ibm.bi.dml.parser.Expression current,
-// com.ibm.bi.dml.parser.Expression left,
-// com.ibm.bi.dml.parser.Expression right, String opStr) {
-// com.ibm.bi.dml.parser.Expression.BooleanOp bop = com.ibm.bi.dml.parser.Expression.getBooleanOp(opStr);
-// current = new com.ibm.bi.dml.parser.BooleanExpression(bop);
-// ((com.ibm.bi.dml.parser.BooleanExpression)current).setLeft(left);
-// ((com.ibm.bi.dml.parser.BooleanExpression)current).setRight(right);
-// ((com.ibm.bi.dml.parser.BooleanExpression)current).setFilename(DmlSyntacticErrorListener.currentFileName.peek());
-// }
-
- public boolean validateBuiltinFunctions(FunctionCallAssignmentStatementContext ctx) {
- String functionName = ctx.name.getText().replaceAll(" ", "").trim();
- if(functionName.compareTo("write") == 0 || functionName.compareTo(DMLProgram.DEFAULT_NAMESPACE + ".write") == 0) {
- return validateBuiltinWriteFunction(ctx);
- }
- return true;
- }
-
- private boolean validateBuiltinWriteFunction(FunctionCallAssignmentStatementContext ctx) {
-
- return true;
- }
-
- public ArrayList<com.ibm.bi.dml.parser.ParameterExpression> getParameterExpressionList(List<ParameterizedExpressionContext> paramExprs) {
- ArrayList<com.ibm.bi.dml.parser.ParameterExpression> retVal = new ArrayList<com.ibm.bi.dml.parser.ParameterExpression>();
- for(ParameterizedExpressionContext ctx : paramExprs) {
- String paramName = null;
- if(ctx.paramName != null && ctx.paramName.getText() != null && !ctx.paramName.getText().isEmpty()) {
- paramName = ctx.paramName.getText();
- }
- com.ibm.bi.dml.parser.ParameterExpression myArg = new com.ibm.bi.dml.parser.ParameterExpression(paramName, ctx.paramVal.info.expr);
- retVal.add(myArg);
- }
- return retVal;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/parser/python/StatementInfo.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/parser/python/StatementInfo.java b/src/main/java/com/ibm/bi/dml/parser/python/StatementInfo.java
deleted file mode 100644
index 82417a5..0000000
--- a/src/main/java/com/ibm/bi/dml/parser/python/StatementInfo.java
+++ /dev/null
@@ -1,13 +0,0 @@
-package com.ibm.bi.dml.parser.python;
-
-/**
- * This class exists solely to prevent compiler warnings.
- *
- * <p>
- * The ExpressionInfo and StatementInfo classes are shared among both parsers
- * (R-like and Python-like dialects), and Antlr-generated code assumes that
- * these classes are present in the parser's namespace.
- */
-class StatementInfo extends com.ibm.bi.dml.parser.antlr4.StatementInfo {
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/DMLRuntimeException.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/DMLRuntimeException.java b/src/main/java/com/ibm/bi/dml/runtime/DMLRuntimeException.java
deleted file mode 100644
index 2bee04c..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/DMLRuntimeException.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime;
-
-import com.ibm.bi.dml.api.DMLException;
-
-/**
- * This exception should be thrown to flag runtime errors -- DML equivalent to java.lang.RuntimeException.
- */
-public class DMLRuntimeException extends DMLException
-{
-
- private static final long serialVersionUID = 1L;
-
- public DMLRuntimeException(String string) {
- super(string);
- }
-
- public DMLRuntimeException(Exception e) {
- super(e);
- }
-
- public DMLRuntimeException(String string, Exception ex){
- super(string,ex);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/DMLScriptException.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/DMLScriptException.java b/src/main/java/com/ibm/bi/dml/runtime/DMLScriptException.java
deleted file mode 100644
index 8caa3e4..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/DMLScriptException.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime;
-
-
-/**
- * This exception should be thrown to flag DML Script errors.
- */
-public class DMLScriptException extends DMLRuntimeException
-{
-
- private static final long serialVersionUID = 1L;
-
- //prevent string concatenation of classname w/ stop message
- private DMLScriptException(Exception e) {
- super(e);
- }
-
- private DMLScriptException(String string, Exception ex){
- super(string,ex);
- }
-
- /**
- * This is the only valid constructor for DMLScriptException.
- *
- * @param string
- */
- public DMLScriptException(String msg) {
- super(msg);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/DMLUnsupportedOperationException.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/DMLUnsupportedOperationException.java b/src/main/java/com/ibm/bi/dml/runtime/DMLUnsupportedOperationException.java
deleted file mode 100644
index c968a03..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/DMLUnsupportedOperationException.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime;
-
-import com.ibm.bi.dml.api.DMLException;
-
-public class DMLUnsupportedOperationException extends DMLException
-{
-
- private static final long serialVersionUID = 1L;
-
- public DMLUnsupportedOperationException(String string) {
- super(string);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/controlprogram/ExternalFunctionProgramBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/ExternalFunctionProgramBlock.java b/src/main/java/com/ibm/bi/dml/runtime/controlprogram/ExternalFunctionProgramBlock.java
deleted file mode 100644
index b48472c..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/ExternalFunctionProgramBlock.java
+++ /dev/null
@@ -1,1064 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime.controlprogram;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.StringTokenizer;
-import java.util.TreeMap;
-
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.conf.DMLConfig;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.lops.ReBlock;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.DMLTranslator;
-import com.ibm.bi.dml.parser.DataIdentifier;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.parser.ExternalFunctionStatement;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.caching.CacheException;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.IDSequence;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.MRJobInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.BooleanObject;
-import com.ibm.bi.dml.runtime.instructions.cp.Data;
-import com.ibm.bi.dml.runtime.instructions.cp.DoubleObject;
-import com.ibm.bi.dml.runtime.instructions.cp.IntObject;
-import com.ibm.bi.dml.runtime.instructions.cp.ScalarObject;
-import com.ibm.bi.dml.runtime.instructions.cp.StringObject;
-import com.ibm.bi.dml.runtime.instructions.cp.VariableCPInstruction;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.MatrixFormatMetaData;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.udf.ExternalFunctionInvocationInstruction;
-import com.ibm.bi.dml.udf.FunctionParameter;
-import com.ibm.bi.dml.udf.Matrix;
-import com.ibm.bi.dml.udf.PackageFunction;
-import com.ibm.bi.dml.udf.PackageRuntimeException;
-import com.ibm.bi.dml.udf.Scalar;
-import com.ibm.bi.dml.udf.FunctionParameter.FunctionParameterType;
-import com.ibm.bi.dml.udf.BinaryObject;
-import com.ibm.bi.dml.udf.Scalar.ScalarValueType;
-
-public class ExternalFunctionProgramBlock extends FunctionProgramBlock
-{
-
- protected static final IDSequence _idSeq = new IDSequence();
-
- protected String _baseDir = null;
-
- ArrayList<Instruction> block2CellInst;
- ArrayList<Instruction> cell2BlockInst;
-
- // holds other key value parameters specified in function declaration
- protected HashMap<String, String> _otherParams;
-
- protected HashMap<String, String> _unblockedFileNames;
- protected HashMap<String, String> _blockedFileNames;
-
- protected long _runID = -1; //ID for block of statements
-
- /**
- * Constructor that also provides otherParams that are needed for external
- * functions. Remaining parameters will just be passed to constructor for
- * function program block.
- *
- * @param eFuncStat
- * @throws DMLRuntimeException
- */
- protected ExternalFunctionProgramBlock(Program prog,
- ArrayList<DataIdentifier> inputParams,
- ArrayList<DataIdentifier> outputParams,
- String baseDir) throws DMLRuntimeException
- {
- super(prog, inputParams, outputParams);
- _baseDir = baseDir;
-
- //NOTE: no need to setup nimble queue for CP external functions
- }
-
- public ExternalFunctionProgramBlock(Program prog,
- ArrayList<DataIdentifier> inputParams,
- ArrayList<DataIdentifier> outputParams,
- HashMap<String, String> otherParams,
- String baseDir) throws DMLRuntimeException {
-
- super(prog, inputParams, outputParams);
- _baseDir = baseDir;
-
- // copy other params
- _otherParams = new HashMap<String, String>();
- _otherParams.putAll(otherParams);
-
- _unblockedFileNames = new HashMap<String, String>();
- _blockedFileNames = new HashMap<String, String>();
-
- // generate instructions
- createInstructions();
- }
-
- private void changeTmpInput( long id, ExecutionContext ec )
- {
- ArrayList<DataIdentifier> inputParams = getInputParams();
- block2CellInst = getBlock2CellInstructions(inputParams, _unblockedFileNames);
-
- //post processing FUNCTION PATCH
- for( String var : _skipInReblock )
- {
- Data dat = ec.getVariable(var);
- if( dat instanceof MatrixObject )
- _unblockedFileNames.put(var, ((MatrixObject)dat).getFileName());
- }
- }
-
- /**
- * It is necessary to change the local temporary files as only file handles are passed out
- * by the external function program block.
- *
- *
- * @param id
- */
- private void changeTmpOutput( long id )
- {
- ArrayList<DataIdentifier> outputParams = getOutputParams();
- cell2BlockInst = getCell2BlockInstructions(outputParams, _blockedFileNames);
- }
-
- /**
- *
- * @return
- */
- public String getBaseDir()
- {
- return _baseDir;
- }
-
- /**
- * Method to be invoked to execute instructions for the external function
- * invocation
- * @throws DMLRuntimeException
- */
- @Override
- public void execute(ExecutionContext ec)
- throws DMLRuntimeException
- {
- _runID = _idSeq.getNextID();
-
- changeTmpInput( _runID, ec );
- changeTmpOutput( _runID );
-
- // export input variables to HDFS (see RunMRJobs)
- ArrayList<DataIdentifier> inputParams = null;
-
- try {
- inputParams = getInputParams();
- for(DataIdentifier di : inputParams ) {
- Data d = ec.getVariable(di.getName());
- if ( d.getDataType() == DataType.MATRIX ) {
- MatrixObject inputObj = (MatrixObject) d;
- inputObj.exportData();
- }
- }
- }
- catch (Exception e){
- throw new PackageRuntimeException(this.printBlockErrorLocation() + "Error exporting input variables to HDFS", e);
- }
-
- // convert block to cell
- if( block2CellInst != null )
- {
- ArrayList<Instruction> tempInst = new ArrayList<Instruction>();
- tempInst.addAll(block2CellInst);
- try {
- this.executeInstructions(tempInst,ec);
- } catch (Exception e) {
-
- throw new PackageRuntimeException(this.printBlockErrorLocation() + "Error executing "
- + tempInst.toString(), e);
- }
- }
-
- // now execute package function
- for (int i = 0; i < _inst.size(); i++)
- {
- try {
- if (_inst.get(i) instanceof ExternalFunctionInvocationInstruction)
- executeInstruction(ec, (ExternalFunctionInvocationInstruction) _inst.get(i));
- }
- catch(Exception e) {
- throw new PackageRuntimeException(this.printBlockErrorLocation() +
- "Failed to execute instruction " + _inst.get(i).toString(), e);
- }
- }
-
- // convert cell to block
- if( cell2BlockInst != null )
- {
- ArrayList<Instruction> tempInst = new ArrayList<Instruction>();
- try {
- tempInst.clear();
- tempInst.addAll(cell2BlockInst);
- this.executeInstructions(tempInst, ec);
- } catch (Exception e) {
-
- throw new PackageRuntimeException(this.printBlockErrorLocation() + "Failed to execute instruction "
- + cell2BlockInst.toString(), e);
- }
- }
-
- // check return values
- checkOutputParameters(ec.getVariables());
- }
-
- /**
- * Given a list of parameters as data identifiers, returns a string
- * representation.
- *
- * @param params
- * @return
- */
-
- protected String getParameterString(ArrayList<DataIdentifier> params) {
- String parameterString = "";
-
- for (int i = 0; i < params.size(); i++) {
- if (i != 0)
- parameterString += ",";
-
- DataIdentifier param = params.get(i);
-
- if (param.getDataType() == DataType.MATRIX) {
- String s = getDataTypeString(DataType.MATRIX) + ":";
- s = s + "" + param.getName() + "" + ":";
- s = s + getValueTypeString(param.getValueType());
- parameterString += s;
- continue;
- }
-
- if (param.getDataType() == DataType.SCALAR) {
- String s = getDataTypeString(DataType.SCALAR) + ":";
- s = s + "" + param.getName() + "" + ":";
- s = s + getValueTypeString(param.getValueType());
- parameterString += s;
- continue;
- }
-
- if (param.getDataType() == DataType.OBJECT) {
- String s = getDataTypeString(DataType.OBJECT) + ":";
- s = s + "" + param.getName() + "" + ":";
- parameterString += s;
- continue;
- }
- }
-
- return parameterString;
- }
-
- /**
- * method to get instructions
- */
- protected void createInstructions() {
-
- _inst = new ArrayList<Instruction>();
-
- // unblock all input matrices
- block2CellInst = getBlock2CellInstructions(getInputParams(),_unblockedFileNames);
-
- // assemble information provided through keyvalue pairs
- String className = _otherParams.get(ExternalFunctionStatement.CLASS_NAME);
- String configFile = _otherParams.get(ExternalFunctionStatement.CONFIG_FILE);
-
- // class name cannot be null, however, configFile and execLocation can
- // be null
- if (className == null)
- throw new PackageRuntimeException(this.printBlockErrorLocation() + ExternalFunctionStatement.CLASS_NAME + " not provided!");
-
- // assemble input and output param strings
- String inputParameterString = getParameterString(getInputParams());
- String outputParameterString = getParameterString(getOutputParams());
-
- // generate instruction
- ExternalFunctionInvocationInstruction einst = new ExternalFunctionInvocationInstruction(
- className, configFile, inputParameterString,
- outputParameterString);
-
- if (getInputParams().size() > 0)
- einst.setLocation(getInputParams().get(0));
- else if (getOutputParams().size() > 0)
- einst.setLocation(getOutputParams().get(0));
- else
- einst.setLocation(this._beginLine, this._endLine, this._beginColumn, this._endColumn);
-
- _inst.add(einst);
-
- // block output matrices
- cell2BlockInst = getCell2BlockInstructions(getOutputParams(),_blockedFileNames);
- }
-
-
- /**
- * Method to generate a reblock job to convert the cell representation into block representation
- * @param outputParams
- * @param blockedFileNames
- * @return
- */
- private ArrayList<Instruction> getCell2BlockInstructions(
- ArrayList<DataIdentifier> outputParams,
- HashMap<String, String> blockedFileNames) {
-
- ArrayList<Instruction> c2binst = null;
-
- //list of matrices that need to be reblocked
- ArrayList<DataIdentifier> matrices = new ArrayList<DataIdentifier>();
- ArrayList<DataIdentifier> matricesNoReblock = new ArrayList<DataIdentifier>();
-
- // identify outputs that are matrices
- for (int i = 0; i < outputParams.size(); i++) {
- if (outputParams.get(i).getDataType() == DataType.MATRIX) {
- if( _skipOutReblock.contains(outputParams.get(i).getName()) )
- matricesNoReblock.add(outputParams.get(i));
- else
- matrices.add(outputParams.get(i));
- }
- }
-
- if( !matrices.isEmpty() )
- {
- c2binst = new ArrayList<Instruction>();
- MRJobInstruction reblkInst = new MRJobInstruction(JobType.REBLOCK);
- TreeMap<Integer, ArrayList<String>> MRJobLineNumbers = null;
- if(DMLScript.ENABLE_DEBUG_MODE) {
- MRJobLineNumbers = new TreeMap<Integer, ArrayList<String>>();
- }
-
- ArrayList<String> inLabels = new ArrayList<String>();
- ArrayList<String> outLabels = new ArrayList<String>();
- String[] outputs = new String[matrices.size()];
- byte[] resultIndex = new byte[matrices.size()];
- String reblock = "";
- String reblockStr = ""; //Keep a copy of a single MR reblock instruction
-
- String scratchSpaceLoc = ConfigurationManager.getConfig().getTextValue(DMLConfig.SCRATCH_SPACE);
-
- try {
- // create a RBLK job that transforms each output matrix from cell to block
- for (int i = 0; i < matrices.size(); i++) {
- inLabels.add(matrices.get(i).getName());
- outLabels.add(matrices.get(i).getName() + "_extFnOutput");
- outputs[i] = scratchSpaceLoc +
- Lop.FILE_SEPARATOR + Lop.PROCESS_PREFIX + DMLScript.getUUID() + Lop.FILE_SEPARATOR +
- _otherParams.get(ExternalFunctionStatement.CLASS_NAME) + _runID + "_" + i + "Output";
- blockedFileNames.put(matrices.get(i).getName(), outputs[i]);
- resultIndex[i] = (byte) i; // (matrices.size()+i);
-
- if (i > 0)
- reblock += Lop.INSTRUCTION_DELIMITOR;
-
- reblock += "MR" + ReBlock.OPERAND_DELIMITOR + "rblk" + ReBlock.OPERAND_DELIMITOR +
- i + ReBlock.DATATYPE_PREFIX + matrices.get(i).getDataType() + ReBlock.VALUETYPE_PREFIX + matrices.get(i).getValueType() + ReBlock.OPERAND_DELIMITOR +
- i + ReBlock.DATATYPE_PREFIX + matrices.get(i).getDataType() + ReBlock.VALUETYPE_PREFIX + matrices.get(i).getValueType() + ReBlock.OPERAND_DELIMITOR +
- DMLTranslator.DMLBlockSize + ReBlock.OPERAND_DELIMITOR + DMLTranslator.DMLBlockSize + ReBlock.OPERAND_DELIMITOR + "true";
-
- if(DMLScript.ENABLE_DEBUG_MODE) {
- //Create a copy of reblock instruction but as a single instruction (FOR DEBUGGER)
- reblockStr = "MR" + ReBlock.OPERAND_DELIMITOR + "rblk" + ReBlock.OPERAND_DELIMITOR +
- i + ReBlock.DATATYPE_PREFIX + matrices.get(i).getDataType() + ReBlock.VALUETYPE_PREFIX + matrices.get(i).getValueType() + ReBlock.OPERAND_DELIMITOR +
- i + ReBlock.DATATYPE_PREFIX + matrices.get(i).getDataType() + ReBlock.VALUETYPE_PREFIX + matrices.get(i).getValueType() + ReBlock.OPERAND_DELIMITOR +
- DMLTranslator.DMLBlockSize + ReBlock.OPERAND_DELIMITOR + DMLTranslator.DMLBlockSize + ReBlock.OPERAND_DELIMITOR + "true";
- //Set MR reblock instruction line number (FOR DEBUGGER)
- if (!MRJobLineNumbers.containsKey(matrices.get(i).getBeginLine())) {
- MRJobLineNumbers.put(matrices.get(i).getBeginLine(), new ArrayList<String>());
- }
- MRJobLineNumbers.get(matrices.get(i).getBeginLine()).add(reblockStr);
- }
- // create metadata instructions to populate symbol table
- // with variables that hold blocked matrices
-
- /*StringBuilder mtdInst = new StringBuilder();
- mtdInst.append("CP" + Lops.OPERAND_DELIMITOR + "createvar");
- mtdInst.append(Lops.OPERAND_DELIMITOR + outLabels.get(i) + Lops.DATATYPE_PREFIX + matrices.get(i).getDataType() + Lops.VALUETYPE_PREFIX + matrices.get(i).getValueType());
- mtdInst.append(Lops.OPERAND_DELIMITOR + outputs[i] + Lops.DATATYPE_PREFIX + DataType.SCALAR + Lops.VALUETYPE_PREFIX + ValueType.STRING);
- mtdInst.append(Lops.OPERAND_DELIMITOR + OutputInfo.outputInfoToString(OutputInfo.BinaryBlockOutputInfo) ) ;
- c2binst.add(CPInstructionParser.parseSingleInstruction(mtdInst.toString()));*/
- Instruction createInst = VariableCPInstruction.prepareCreateVariableInstruction(outLabels.get(i), outputs[i], false, OutputInfo.outputInfoToString(OutputInfo.BinaryBlockOutputInfo));
-
- createInst.setLocation(matrices.get(i));
-
- c2binst.add(createInst);
-
- }
-
- reblkInst.setReBlockInstructions(inLabels.toArray(new String[inLabels.size()]), "", reblock, "",
- outLabels.toArray(new String[inLabels.size()]), resultIndex, 1, 1);
- c2binst.add(reblkInst);
-
- // generate instructions that rename the output variables of REBLOCK job
- Instruction cpInst = null, rmInst = null;
- for (int i = 0; i < matrices.size(); i++) {
- cpInst = VariableCPInstruction.prepareCopyInstruction(outLabels.get(i), matrices.get(i).getName());
- rmInst = VariableCPInstruction.prepareRemoveInstruction(outLabels.get(i));
-
- cpInst.setLocation(matrices.get(i));
- rmInst.setLocation(matrices.get(i));
-
- c2binst.add(cpInst);
- c2binst.add(rmInst);
- //c2binst.add(CPInstructionParser.parseSingleInstruction("CP" + Lops.OPERAND_DELIMITOR + "cpvar"+Lops.OPERAND_DELIMITOR+ outLabels.get(i) + Lops.OPERAND_DELIMITOR + matrices.get(i).getName()));
- }
- } catch (Exception e) {
- throw new PackageRuntimeException(this.printBlockErrorLocation() + "error generating instructions", e);
- }
-
- //LOGGING instructions
- if (LOG.isTraceEnabled()){
- LOG.trace("\n--- Cell-2-Block Instructions ---");
- for(Instruction i : c2binst) {
- LOG.trace(i.toString());
- }
- LOG.trace("----------------------------------");
- }
-
- }
-
- return c2binst; //null if no output matrices
- }
-
- /**
- * Method to generate instructions to convert input matrices from block to
- * cell. We generate a GMR job here.
- *
- * @param inputParams
- * @return
- */
- private ArrayList<Instruction> getBlock2CellInstructions(
- ArrayList<DataIdentifier> inputParams,
- HashMap<String, String> unBlockedFileNames) {
-
- ArrayList<Instruction> b2cinst = null;
-
- //list of input matrices
- ArrayList<DataIdentifier> matrices = new ArrayList<DataIdentifier>();
- ArrayList<DataIdentifier> matricesNoReblock = new ArrayList<DataIdentifier>();
-
- // find all inputs that are matrices
- for (int i = 0; i < inputParams.size(); i++) {
- if (inputParams.get(i).getDataType() == DataType.MATRIX) {
- if( _skipInReblock.contains(inputParams.get(i).getName()) )
- matricesNoReblock.add(inputParams.get(i));
- else
- matrices.add(inputParams.get(i));
- }
- }
-
- if( !matrices.isEmpty() )
- {
- b2cinst = new ArrayList<Instruction>();
- MRJobInstruction gmrInst = new MRJobInstruction(JobType.GMR);
- TreeMap<Integer, ArrayList<String>> MRJobLineNumbers = null;
- if(DMLScript.ENABLE_DEBUG_MODE) {
- MRJobLineNumbers = new TreeMap<Integer, ArrayList<String>>();
- }
- String gmrStr="";
- ArrayList<String> inLabels = new ArrayList<String>();
- ArrayList<String> outLabels = new ArrayList<String>();
- String[] outputs = new String[matrices.size()];
- byte[] resultIndex = new byte[matrices.size()];
-
- String scratchSpaceLoc = ConfigurationManager.getConfig().getTextValue(DMLConfig.SCRATCH_SPACE);
-
-
- try {
- // create a GMR job that transforms each of these matrices from block to cell
- for (int i = 0; i < matrices.size(); i++) {
-
- //inputs[i] = "##" + matrices.get(i).getName() + "##";
- //inputInfo[i] = binBlockInputInfo;
- //outputInfo[i] = textCellOutputInfo;
- //numRows[i] = numCols[i] = numRowsPerBlock[i] = numColsPerBlock[i] = -1;
- //resultDimsUnknown[i] = 1;
-
- inLabels.add(matrices.get(i).getName());
- outLabels.add(matrices.get(i).getName()+"_extFnInput");
- resultIndex[i] = (byte) i; //(matrices.size()+i);
-
- outputs[i] = scratchSpaceLoc +
- Lop.FILE_SEPARATOR + Lop.PROCESS_PREFIX + DMLScript.getUUID() + Lop.FILE_SEPARATOR +
- _otherParams.get(ExternalFunctionStatement.CLASS_NAME) + _runID + "_" + i + "Input";
- unBlockedFileNames.put(matrices.get(i).getName(), outputs[i]);
-
- if(DMLScript.ENABLE_DEBUG_MODE) {
- //Create a dummy gmr instruction (FOR DEBUGGER)
- gmrStr = "MR" + Lop.OPERAND_DELIMITOR + "gmr" + Lop.OPERAND_DELIMITOR +
- i + Lop.DATATYPE_PREFIX + matrices.get(i).getDataType() + Lop.VALUETYPE_PREFIX + matrices.get(i).getValueType() + Lop.OPERAND_DELIMITOR +
- i + Lop.DATATYPE_PREFIX + matrices.get(i).getDataType() + Lop.VALUETYPE_PREFIX + matrices.get(i).getValueType() + Lop.OPERAND_DELIMITOR +
- DMLTranslator.DMLBlockSize + Lop.OPERAND_DELIMITOR + DMLTranslator.DMLBlockSize;
-
- //Set MR gmr instruction line number (FOR DEBUGGER)
- if (!MRJobLineNumbers.containsKey(matrices.get(i).getBeginLine())) {
- MRJobLineNumbers.put(matrices.get(i).getBeginLine(), new ArrayList<String>());
- }
- MRJobLineNumbers.get(matrices.get(i).getBeginLine()).add(gmrStr);
- }
- // create metadata instructions to populate symbol table
- // with variables that hold unblocked matrices
-
- /*StringBuilder mtdInst = new StringBuilder();
- mtdInst.append("CP" + Lops.OPERAND_DELIMITOR + "createvar");
- mtdInst.append(Lops.OPERAND_DELIMITOR + outLabels.get(i) + Lops.DATATYPE_PREFIX + matrices.get(i).getDataType() + Lops.VALUETYPE_PREFIX + matrices.get(i).getValueType());
- mtdInst.append(Lops.OPERAND_DELIMITOR + outputs[i] + Lops.DATATYPE_PREFIX + DataType.SCALAR + Lops.VALUETYPE_PREFIX + ValueType.STRING);
- mtdInst.append(Lops.OPERAND_DELIMITOR + OutputInfo.outputInfoToString(OutputInfo.TextCellOutputInfo) ) ;
- b2cinst.add(CPInstructionParser.parseSingleInstruction(mtdInst.toString()));*/
-
- Instruction createInst = VariableCPInstruction.prepareCreateVariableInstruction(outLabels.get(i), outputs[i], false, OutputInfo.outputInfoToString(OutputInfo.TextCellOutputInfo));
-
- createInst.setLocation(matrices.get(i));
-
- b2cinst.add(createInst);
- }
-
- // Finally, generate GMR instruction that performs block2cell conversion
- gmrInst.setGMRInstructions(inLabels.toArray(new String[inLabels.size()]), "", "", "", "",
- outLabels.toArray(new String[outLabels.size()]), resultIndex, 0, 1);
-
- b2cinst.add(gmrInst);
-
- // generate instructions that rename the output variables of GMR job
- Instruction cpInst=null, rmInst=null;
- for (int i = 0; i < matrices.size(); i++) {
- cpInst = VariableCPInstruction.prepareCopyInstruction(outLabels.get(i), matrices.get(i).getName());
- rmInst = VariableCPInstruction.prepareRemoveInstruction(outLabels.get(i));
-
- cpInst.setLocation(matrices.get(i));
- rmInst.setLocation(matrices.get(i));
-
- b2cinst.add(cpInst);
- b2cinst.add(rmInst);
- }
- } catch (Exception e) {
- throw new PackageRuntimeException(e);
- }
-
- //LOG instructions
- if (LOG.isTraceEnabled()){
- LOG.trace("\n--- Block-2-Cell Instructions ---");
- for(Instruction i : b2cinst) {
- LOG.trace(i.toString());
- }
- LOG.trace("----------------------------------");
- }
- }
-
- //BEGIN FUNCTION PATCH
- if( !matricesNoReblock.isEmpty() )
- {
- //if( b2cinst==null )
- // b2cinst = new ArrayList<Instruction>();
-
- for( int i=0; i<matricesNoReblock.size(); i++ )
- {
- String scratchSpaceLoc = ConfigurationManager.getConfig().getTextValue(DMLConfig.SCRATCH_SPACE);
-
- try{
- String filename = scratchSpaceLoc +
- Lop.FILE_SEPARATOR + Lop.PROCESS_PREFIX + DMLScript.getUUID() + Lop.FILE_SEPARATOR +
- _otherParams.get(ExternalFunctionStatement.CLASS_NAME) + _runID + "_" + i + "Input";
- //String outLabel = matricesNoReblock.get(i).getName()+"_extFnInput";
- //Instruction createInst = VariableCPInstruction.prepareCreateVariableInstruction(outLabel, filename, false, OutputInfo.outputInfoToString(OutputInfo.TextCellOutputInfo));
- //Instruction cpInst = VariableCPInstruction.prepareCopyInstruction( matricesNoReblock.get(i).getName(), outLabel);
-
- unBlockedFileNames.put(matricesNoReblock.get(i).getName(), filename); //
-
- //b2cinst.add(createInst);
- //b2cinst.add(cpInst);
- }
- catch (Exception e) {
- throw new PackageRuntimeException(e);
- }
-
- }
- }
- //END FUNCTION PATCH
-
- return b2cinst; //null if no input matrices
- }
-
- /**
- * Method to execute an external function invocation instruction.
- *
- * @param inst
- * @param dQueue
- * @throws NimbleCheckedRuntimeException
- * @throws DMLRuntimeException
- */
- @SuppressWarnings("unchecked")
- public void executeInstruction(ExecutionContext ec, ExternalFunctionInvocationInstruction inst)
- throws DMLRuntimeException
- {
- String className = inst.getClassName();
- String configFile = inst.getConfigFile();
-
- if (className == null)
- throw new PackageRuntimeException(this.printBlockErrorLocation() + "Class name can't be null");
-
- // create instance of package function.
- Object o;
- try
- {
- Class<Instruction> cla = (Class<Instruction>) Class.forName(className);
- o = cla.newInstance();
- }
- catch (Exception e)
- {
- throw new PackageRuntimeException(this.printBlockErrorLocation() + "Error generating package function object " ,e );
- }
-
- if (!(o instanceof PackageFunction))
- throw new PackageRuntimeException(this.printBlockErrorLocation() + "Class is not of type PackageFunction");
-
- PackageFunction func = (PackageFunction) o;
-
- // add inputs to this package function based on input parameter
- // and their mappings.
- setupInputs(func, inst.getInputParams(), ec.getVariables());
- func.setConfiguration(configFile);
- func.setBaseDir(_baseDir);
-
- //executes function
- func.execute();
-
- // verify output of function execution matches declaration
- // and add outputs to variableMapping and Metadata
- verifyAndAttachOutputs(ec, func, inst.getOutputParams());
- }
-
- /**
- * Method to verify that function outputs match with declared outputs
- *
- * @param returnFunc
- * @param outputParams
- * @throws DMLRuntimeException
- */
- protected void verifyAndAttachOutputs(ExecutionContext ec, PackageFunction returnFunc,
- String outputParams) throws DMLRuntimeException {
-
- ArrayList<String> outputs = getParameters(outputParams);
- // make sure they are of equal size first
-
- if (outputs.size() != returnFunc.getNumFunctionOutputs()) {
- throw new PackageRuntimeException(
- "Number of function outputs ("+returnFunc.getNumFunctionOutputs()+") " +
- "does not match with declaration ("+outputs.size()+").");
- }
-
- // iterate over each output and verify that type matches
- for (int i = 0; i < outputs.size(); i++) {
- StringTokenizer tk = new StringTokenizer(outputs.get(i), ":");
- ArrayList<String> tokens = new ArrayList<String>();
- while (tk.hasMoreTokens()) {
- tokens.add(tk.nextToken());
- }
-
- if (returnFunc.getFunctionOutput(i).getType() == FunctionParameterType.Matrix) {
- Matrix m = (Matrix) returnFunc.getFunctionOutput(i);
-
- if (!(tokens.get(0)
- .compareTo(getFunctionParameterDataTypeString(FunctionParameterType.Matrix)) == 0)
- || !(tokens.get(2).compareTo(
- getMatrixValueTypeString(m.getValueType())) == 0)) {
- throw new PackageRuntimeException(
- "Function output '"+outputs.get(i)+"' does not match with declaration.");
- }
-
- // add result to variableMapping
- String varName = tokens.get(1);
- MatrixObject newVar = createOutputMatrixObject( m );
- newVar.setVarName(varName);
-
- /* cleanup not required because done at central position (FunctionCallCPInstruction)
- MatrixObjectNew oldVar = (MatrixObjectNew)getVariable(varName);
- if( oldVar!=null )
- oldVar.clearData();*/
-
- //getVariables().put(varName, newVar); //put/override in local symbol table
- ec.setVariable(varName, newVar);
-
- continue;
- }
-
- if (returnFunc.getFunctionOutput(i).getType() == FunctionParameterType.Scalar) {
- Scalar s = (Scalar) returnFunc.getFunctionOutput(i);
-
- if (!tokens.get(0).equals(getFunctionParameterDataTypeString(FunctionParameterType.Scalar))
- || !tokens.get(2).equals(
- getScalarValueTypeString(s.getScalarType()))) {
- throw new PackageRuntimeException(
- "Function output '"+outputs.get(i)+"' does not match with declaration.");
- }
-
- // allocate and set appropriate object based on type
- ScalarObject scalarObject = null;
- ScalarValueType type = s.getScalarType();
- switch (type) {
- case Integer:
- scalarObject = new IntObject(tokens.get(1),
- Long.parseLong(s.getValue()));
- break;
- case Double:
- scalarObject = new DoubleObject(tokens.get(1),
- Double.parseDouble(s.getValue()));
- break;
- case Boolean:
- scalarObject = new BooleanObject(tokens.get(1),
- Boolean.parseBoolean(s.getValue()));
- break;
- case Text:
- scalarObject = new StringObject(tokens.get(1), s.getValue());
- break;
- default:
- throw new PackageRuntimeException(
- "Unknown scalar value type '"+type+"' of output '"+outputs.get(i)+"'.");
- }
-
- //this.getVariables().put(tokens.get(1), scalarObject);
- ec.setVariable(tokens.get(1), scalarObject);
- continue;
- }
-
- if (returnFunc.getFunctionOutput(i).getType() == FunctionParameterType.Object) {
- if (!tokens.get(0).equals(getFunctionParameterDataTypeString(FunctionParameterType.Object))) {
- throw new PackageRuntimeException(
- "Function output '"+outputs.get(i)+"' does not match with declaration.");
- }
-
- throw new PackageRuntimeException(
- "Object types not yet supported");
-
- // continue;
- }
-
- throw new PackageRuntimeException(
- "Unknown data type '"+returnFunc.getFunctionOutput(i).getType()+"' " +
- "of output '"+outputs.get(i)+"'.");
- }
- }
-
- protected MatrixObject createOutputMatrixObject( Matrix m )
- throws CacheException
- {
- MatrixCharacteristics mc = new MatrixCharacteristics(m.getNumRows(),m.getNumCols(), DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize);
- MatrixFormatMetaData mfmd = new MatrixFormatMetaData(mc, OutputInfo.TextCellOutputInfo, InputInfo.TextCellInputInfo);
- return new MatrixObject(ValueType.DOUBLE, m.getFilePath(), mfmd);
- }
-
- /**
- * Method to get string representation of scalar value type
- *
- * @param scalarType
- * @return
- */
-
- protected String getScalarValueTypeString(ScalarValueType scalarType) {
-
- if (scalarType.equals(ScalarValueType.Double))
- return "Double";
- if (scalarType.equals(ScalarValueType.Integer))
- return "Integer";
- if (scalarType.equals(ScalarValueType.Boolean))
- return "Boolean";
- if (scalarType.equals(ScalarValueType.Text))
- return "String";
-
- throw new PackageRuntimeException("Unknown scalar value type");
- }
-
- /**
- * Method to parse inputs, update labels, and add to package function.
- *
- * @param func
- * @param inputParams
- * @param metaData
- * @param variableMapping
- */
- protected void setupInputs (PackageFunction func, String inputParams,
- LocalVariableMap variableMapping) {
-
- ArrayList<String> inputs = getParameters(inputParams);
- ArrayList<FunctionParameter> inputObjects = getInputObjects(inputs, variableMapping);
- func.setNumFunctionInputs(inputObjects.size());
- for (int i = 0; i < inputObjects.size(); i++)
- func.setInput(inputObjects.get(i), i);
-
- }
-
- /**
- * Method to convert string representation of input into function input
- * object.
- *
- * @param inputs
- * @param variableMapping
- * @param metaData
- * @return
- */
-
- protected ArrayList<FunctionParameter> getInputObjects(ArrayList<String> inputs,
- LocalVariableMap variableMapping) {
- ArrayList<FunctionParameter> inputObjects = new ArrayList<FunctionParameter>();
-
- for (int i = 0; i < inputs.size(); i++) {
- ArrayList<String> tokens = new ArrayList<String>();
- StringTokenizer tk = new StringTokenizer(inputs.get(i), ":");
- while (tk.hasMoreTokens()) {
- tokens.add(tk.nextToken());
- }
-
- if (tokens.get(0).equals("Matrix")) {
- String varName = tokens.get(1);
- MatrixObject mobj = (MatrixObject) variableMapping.get(varName);
- MatrixCharacteristics mc = mobj.getMatrixCharacteristics();
- Matrix m = new Matrix(mobj.getFileName(),
- mc.getRows(), mc.getCols(),
- getMatrixValueType(tokens.get(2)));
- modifyInputMatrix(m, mobj);
- inputObjects.add(m);
- }
-
- if (tokens.get(0).equals("Scalar")) {
- String varName = tokens.get(1);
- ScalarObject so = (ScalarObject) variableMapping.get(varName);
- Scalar s = new Scalar(getScalarValueType(tokens.get(2)),
- so.getStringValue());
- inputObjects.add(s);
-
- }
-
- if (tokens.get(0).equals("Object")) {
- String varName = tokens.get(1);
- Object o = variableMapping.get(varName);
- BinaryObject obj = new BinaryObject(o);
- inputObjects.add(obj);
-
- }
- }
-
- return inputObjects;
-
- }
-
- protected void modifyInputMatrix(Matrix m, MatrixObject mobj)
- {
- //do nothing, intended for extensions
- }
-
- /**
- * Converts string representation of scalar value type to enum type
- *
- * @param string
- * @return
- */
- protected ScalarValueType getScalarValueType(String string) {
- if (string.equals("Double"))
- return ScalarValueType.Double;
- if (string.equals("Integer"))
- return ScalarValueType.Integer;
- if (string.equals("Boolean"))
- return ScalarValueType.Boolean;
- if (string.equals("String"))
- return ScalarValueType.Text;
-
- throw new PackageRuntimeException("Unknown scalar type");
-
- }
-
- /**
- * Get string representation of matrix value type
- *
- * @param t
- * @return
- */
-
- protected String getMatrixValueTypeString(Matrix.ValueType t) {
- if (t.equals(Matrix.ValueType.Double))
- return "Double";
-
- if (t.equals(Matrix.ValueType.Integer))
- return "Integer";
-
- throw new PackageRuntimeException("Unknown matrix value type");
- }
-
- /**
- * Converts string representation of matrix value type into enum type
- *
- * @param string
- * @return
- */
-
- protected com.ibm.bi.dml.udf.Matrix.ValueType getMatrixValueType(String string) {
-
- if (string.equals("Double"))
- return Matrix.ValueType.Double;
- if (string.equals("Integer"))
- return Matrix.ValueType.Integer;
-
- throw new PackageRuntimeException("Unknown matrix value type");
-
- }
-
- /**
- * Method to break the comma separated input parameters into an arraylist of
- * parameters
- *
- * @param inputParams
- * @return
- */
- protected ArrayList<String> getParameters(String inputParams) {
- ArrayList<String> inputs = new ArrayList<String>();
-
- StringTokenizer tk = new StringTokenizer(inputParams, ",");
- while (tk.hasMoreTokens()) {
- inputs.add(tk.nextToken());
- }
-
- return inputs;
- }
-
- /**
- * Get string representation for data type
- *
- * @param d
- * @return
- */
- protected String getDataTypeString(DataType d) {
- if (d.equals(DataType.MATRIX))
- return "Matrix";
-
- if (d.equals(DataType.SCALAR))
- return "Scalar";
-
- if (d.equals(DataType.OBJECT))
- return "Object";
-
- throw new PackageRuntimeException("Should never come here");
-
- }
-
- /**
- * Method to get string representation of data type.
- *
- * @param t
- * @return
- */
- protected String getFunctionParameterDataTypeString(FunctionParameterType t) {
- if (t.equals(FunctionParameterType.Matrix))
- return "Matrix";
-
- if (t.equals(FunctionParameterType.Scalar))
- return "Scalar";
-
- if (t.equals(FunctionParameterType.Object))
- return "Object";
-
- throw new PackageRuntimeException("Should never come here");
- }
-
- /**
- * Get string representation of value type
- *
- * @param v
- * @return
- */
- protected String getValueTypeString(ValueType v) {
- if (v.equals(ValueType.DOUBLE))
- return "Double";
-
- if (v.equals(ValueType.INT))
- return "Integer";
-
- if (v.equals(ValueType.BOOLEAN))
- return "Boolean";
-
- if (v.equals(ValueType.STRING))
- return "String";
-
- throw new PackageRuntimeException("Should never come here");
- }
-
- public void printMe() {
- //System.out.println("***** INSTRUCTION BLOCK *****");
- for (Instruction i : this._inst) {
- i.printMe();
- }
- }
-
- public HashMap<String,String> getOtherParams()
- {
- return _otherParams;
- }
-
- public String printBlockErrorLocation(){
- return "ERROR: Runtime error in external function program block generated from external function statement block between lines " + _beginLine + " and " + _endLine + " -- ";
- }
-
-
- /////////////////////////////////////////////////
- // Extension for Global Data Flow Optimization
- // by Mathias Peters
- ///////
-
- //FUNCTION PATCH
-
- private Collection<String> _skipInReblock = new HashSet<String>();
- private Collection<String> _skipOutReblock = new HashSet<String>();
-
- public void setSkippedReblockLists( Collection<String> varsIn, Collection<String> varsOut )
- {
- _skipInReblock.clear();
- _skipOutReblock.clear();
-
- if( varsIn!=null || varsOut!=null )
- {
- if( varsIn != null )
- _skipInReblock.addAll(varsIn);
- if( varsOut != null )
- _skipOutReblock.addAll(varsOut);
-
- //regenerate instructions
- createInstructions();
- }
- }
-
-
- @Override
- public ArrayList<Instruction> getInstructions()
- {
- ArrayList<Instruction> tmp = new ArrayList<Instruction>();
- if( cell2BlockInst != null )
- tmp.addAll(cell2BlockInst);
- if( block2CellInst != null )
- tmp.addAll(block2CellInst);
- return tmp;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/controlprogram/ExternalFunctionProgramBlockCP.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/ExternalFunctionProgramBlockCP.java b/src/main/java/com/ibm/bi/dml/runtime/controlprogram/ExternalFunctionProgramBlockCP.java
deleted file mode 100644
index ebbd53d..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/ExternalFunctionProgramBlockCP.java
+++ /dev/null
@@ -1,192 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime.controlprogram;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-
-import com.ibm.bi.dml.parser.DMLTranslator;
-import com.ibm.bi.dml.parser.DataIdentifier;
-import com.ibm.bi.dml.parser.ExternalFunctionStatement;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.IDSequence;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.MatrixFormatMetaData;
-import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.udf.ExternalFunctionInvocationInstruction;
-import com.ibm.bi.dml.udf.Matrix;
-import com.ibm.bi.dml.udf.PackageRuntimeException;
-
-/**
- * CP external function program block, that overcomes the need for
- * BlockToCell and CellToBlock MR jobs by changing the contract for an external function.
- * If execlocation="CP", the implementation of an external function must read and write
- * matrices as InputInfo.BinaryBlockInputInfo and OutputInfo.BinaryBlockOutputInfo.
- *
- * Furthermore, it extends ExternalFunctionProgramBlock with a base directory in order
- * to make it parallelizable, even in case of different JVMs. For this purpose every
- * external function must implement a <SET_BASE_DIR> method.
- *
- *
- */
-public class ExternalFunctionProgramBlockCP extends ExternalFunctionProgramBlock
-{
-
- public static String DEFAULT_FILENAME = "ext_funct";
- private static IDSequence _defaultSeq = new IDSequence();
-
- /**
- * Constructor that also provides otherParams that are needed for external
- * functions. Remaining parameters will just be passed to constructor for
- * function program block.
- *
- * @param eFuncStat
- * @throws DMLRuntimeException
- */
- public ExternalFunctionProgramBlockCP(Program prog,
- ArrayList<DataIdentifier> inputParams,
- ArrayList<DataIdentifier> outputParams,
- HashMap<String, String> otherParams,
- String baseDir) throws DMLRuntimeException {
-
- super(prog, inputParams, outputParams, baseDir); //w/o instruction generation
-
- // copy other params
- _otherParams = new HashMap<String, String>();
- _otherParams.putAll(otherParams);
-
- // generate instructions (overwritten)
- createInstructions();
- }
-
- /**
- * Method to be invoked to execute instructions for the external function
- * invocation
- * @throws DMLRuntimeException
- */
- @Override
- public void execute(ExecutionContext ec) throws DMLRuntimeException
- {
- _runID = _idSeq.getNextID();
-
- ExternalFunctionInvocationInstruction inst = null;
-
- // execute package function
- for (int i=0; i < _inst.size(); i++)
- {
- try {
- inst = (ExternalFunctionInvocationInstruction)_inst.get(i);
- executeInstruction( ec, inst );
- }
- catch (Exception e){
- throw new DMLRuntimeException(this.printBlockErrorLocation() + "Error evaluating instruction " + i + " in external function programBlock. inst: " + inst.toString(), e);
- }
- }
-
- // check return values
- checkOutputParameters(ec.getVariables());
- }
-
- /**
- * Executes the external function instruction without the use of NIMBLE tasks.
- *
- * @param inst
- * @throws DMLRuntimeException
- * @throws NimbleCheckedRuntimeException
- */
- @Override
- public void executeInstruction(ExecutionContext ec, ExternalFunctionInvocationInstruction inst)
- throws DMLRuntimeException
- {
- // After removal of nimble, we moved the code of ExternalFunctionProgramBlockCP to
- // ExternalFunctionProgramBlock and hence hence both types of external functions can
- // share the same code path here.
- super.executeInstruction(ec, inst);
- }
-
-
- @Override
- protected void createInstructions()
- {
- _inst = new ArrayList<Instruction>();
-
- // assemble information provided through keyvalue pairs
- String className = _otherParams.get(ExternalFunctionStatement.CLASS_NAME);
- String configFile = _otherParams.get(ExternalFunctionStatement.CONFIG_FILE);
-
- // class name cannot be null, however, configFile and execLocation can be null
- if (className == null)
- throw new PackageRuntimeException(this.printBlockErrorLocation() + ExternalFunctionStatement.CLASS_NAME + " not provided!");
-
- // assemble input and output param strings
- String inputParameterString = getParameterString(getInputParams());
- String outputParameterString = getParameterString(getOutputParams());
-
- // generate instruction
- ExternalFunctionInvocationInstruction einst = new ExternalFunctionInvocationInstruction(
- className, configFile, inputParameterString,
- outputParameterString);
-
- _inst.add(einst);
-
- }
-
- @Override
- protected void modifyInputMatrix(Matrix m, MatrixObject mobj)
- {
- //pass in-memory object to external function
- m.setMatrixObject( mobj );
- }
-
- @Override
- protected MatrixObject createOutputMatrixObject(Matrix m)
- {
- MatrixObject ret = m.getMatrixObject();
-
- if( ret == null ) //otherwise, pass in-memory matrix from extfunct back to invoking program
- {
- MatrixCharacteristics mc = new MatrixCharacteristics(m.getNumRows(),m.getNumCols(), DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize);
- MatrixFormatMetaData mfmd = new MatrixFormatMetaData(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo);
- ret = new MatrixObject(ValueType.DOUBLE, m.getFilePath(), mfmd);
- }
-
- //for allowing in-memory packagesupport matrices w/o filesnames
- if( ret.getFileName().equals( DEFAULT_FILENAME ) )
- {
- ret.setFileName( createDefaultOutputFilePathAndName() );
- }
-
- return ret;
- }
-
-
- public String createDefaultOutputFilePathAndName( )
- {
- return _baseDir + DEFAULT_FILENAME + _defaultSeq.getNextID();
- }
-
- public String printBlockErrorLocation(){
- return "ERROR: Runtime error in external function program block (for CP) generated from external function statement block between lines " + _beginLine + " and " + _endLine + " -- ";
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/controlprogram/ForProgramBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/ForProgramBlock.java b/src/main/java/com/ibm/bi/dml/runtime/controlprogram/ForProgramBlock.java
deleted file mode 100644
index c40420c..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/ForProgramBlock.java
+++ /dev/null
@@ -1,288 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime.controlprogram;
-
-import java.util.ArrayList;
-
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.parser.ForStatementBlock;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLScriptException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.cp.Data;
-import com.ibm.bi.dml.runtime.instructions.cp.IntObject;
-import com.ibm.bi.dml.runtime.instructions.cp.ScalarObject;
-import com.ibm.bi.dml.runtime.util.UtilFunctions;
-import com.ibm.bi.dml.yarn.DMLAppMasterUtils;
-
-public class ForProgramBlock extends ProgramBlock
-{
-
- protected ArrayList<Instruction> _fromInstructions;
- protected ArrayList<Instruction> _toInstructions;
- protected ArrayList<Instruction> _incrementInstructions;
-
- protected ArrayList <Instruction> _exitInstructions ;
- protected ArrayList<ProgramBlock> _childBlocks;
-
- protected String[] _iterablePredicateVars; //from,to,where constants/internal vars not captured via instructions
-
- public void printMe()
- {
- LOG.debug("***** current for block predicate inst: *****");
- LOG.debug("FROM:");
- for (Instruction cp : _fromInstructions){
- cp.printMe();
- }
- LOG.debug("TO:");
- for (Instruction cp : _toInstructions){
- cp.printMe();
- }
- LOG.debug("INCREMENT:");
- for (Instruction cp : _incrementInstructions){
- cp.printMe();
- }
-
- LOG.debug("***** children block inst: *****");
- for (ProgramBlock pb : this._childBlocks){
- pb.printMe();
- }
-
- LOG.debug("***** current block inst exit: *****");
- for (Instruction i : this._exitInstructions) {
- i.printMe();
- }
-
- }
-
-
- public ForProgramBlock(Program prog, String[] iterPredVars) throws DMLRuntimeException
- {
- super(prog);
-
- _exitInstructions = new ArrayList<Instruction>();
- _childBlocks = new ArrayList<ProgramBlock>();
- _iterablePredicateVars = iterPredVars;
- }
-
- public ArrayList<Instruction> getFromInstructions()
- {
- return _fromInstructions;
- }
-
- public void setFromInstructions(ArrayList<Instruction> instructions)
- {
- _fromInstructions = instructions;
- }
-
- public ArrayList<Instruction> getToInstructions()
- {
- return _toInstructions;
- }
-
- public void setToInstructions(ArrayList<Instruction> instructions)
- {
- _toInstructions = instructions;
- }
-
- public ArrayList<Instruction> getIncrementInstructions()
- {
- return _incrementInstructions;
- }
-
- public void setIncrementInstructions(ArrayList<Instruction> instructions)
- {
- _incrementInstructions = instructions;
- }
-
- public void addExitInstruction(Instruction inst){
- _exitInstructions.add(inst);
- }
-
- public ArrayList<Instruction> getExitInstructions(){
- return _exitInstructions;
- }
-
- public void setExitInstructions(ArrayList<Instruction> inst){
- _exitInstructions = inst;
- }
-
-
- public void addProgramBlock(ProgramBlock childBlock) {
- _childBlocks.add(childBlock);
- }
-
- public ArrayList<ProgramBlock> getChildBlocks()
- {
- return _childBlocks;
- }
-
- public void setChildBlocks(ArrayList<ProgramBlock> pbs)
- {
- _childBlocks = pbs;
- }
-
- public String[] getIterablePredicateVars()
- {
- return _iterablePredicateVars;
- }
-
- public void setIterablePredicateVars(String[] iterPredVars)
- {
- _iterablePredicateVars = iterPredVars;
- }
-
- @Override
- public void execute(ExecutionContext ec)
- throws DMLRuntimeException, DMLUnsupportedOperationException
- {
- // add the iterable predicate variable to the variable set
- String iterVarName = _iterablePredicateVars[0];
-
- // evaluate from, to, incr only once (assumption: known at for entry)
- IntObject from = executePredicateInstructions( 1, _fromInstructions, ec );
- IntObject to = executePredicateInstructions( 2, _toInstructions, ec );
- IntObject incr = executePredicateInstructions( 3, _incrementInstructions, ec );
-
- if ( incr.getLongValue() <= 0 ) //would produce infinite loop
- throw new DMLRuntimeException(this.printBlockErrorLocation() + "Expression for increment of variable '" + iterVarName + "' must evaluate to a positive value.");
-
- // initialize iter var to from value
- IntObject iterVar = new IntObject(iterVarName, from.getLongValue() );
-
- // execute for loop
- try
- {
- // run for loop body as long as predicate is true
- // (for supporting dynamic TO, move expression execution to end of while loop)
- while( iterVar.getLongValue() <= to.getLongValue() )
- {
- ec.setVariable(iterVarName, iterVar);
-
- //for all child blocks
- for (int i=0 ; i < this._childBlocks.size() ; i++) {
- ec.updateDebugState( i );
- _childBlocks.get(i).execute(ec);
- }
-
- // update the iterable predicate variable
- if(ec.getVariable(iterVarName) == null || !(ec.getVariable(iterVarName) instanceof IntObject))
- throw new DMLRuntimeException("Iterable predicate variable " + iterVarName + " must remain of type scalar int.");
-
- //increment of iterVar (changes in loop body get discarded)
- iterVar = new IntObject( iterVarName, iterVar.getLongValue()+incr.getLongValue() );
- }
- }
- catch (DMLScriptException e)
- {
- throw e;
- }
- catch (Exception e)
- {
- throw new DMLRuntimeException(printBlockErrorLocation() + "Error evaluating for program block", e);
- }
-
- //execute exit instructions
- try {
- executeInstructions(_exitInstructions, ec);
- }
- catch (Exception e){
- throw new DMLRuntimeException(printBlockErrorLocation() + "Error evaluating for exit instructions", e);
- }
- }
-
- /**
- *
- * @param pos
- * @param instructions
- * @param ec
- * @return
- * @throws DMLRuntimeException
- */
- protected IntObject executePredicateInstructions( int pos, ArrayList<Instruction> instructions, ExecutionContext ec )
- throws DMLRuntimeException
- {
- ScalarObject tmp = null;
- IntObject ret = null;
-
- try
- {
- if( _iterablePredicateVars[pos] != null )
- {
- //probe for scalar variables
- Data ldat = ec.getVariable( _iterablePredicateVars[pos] );
- if( ldat != null && ldat instanceof ScalarObject )
- tmp = (ScalarObject)ldat;
- else //handle literals
- tmp = new IntObject( UtilFunctions.parseToLong(_iterablePredicateVars[pos]) );
- }
- else
- {
- if( _sb!=null )
- {
- if( DMLScript.isActiveAM() ) //set program block specific remote memory
- DMLAppMasterUtils.setupProgramBlockRemoteMaxMemory(this);
-
- ForStatementBlock fsb = (ForStatementBlock)_sb;
- Hop predHops = null;
- boolean recompile = false;
- if (pos == 1){
- predHops = fsb.getFromHops();
- recompile = fsb.requiresFromRecompilation();
- }
- else if (pos == 2) {
- predHops = fsb.getToHops();
- recompile = fsb.requiresToRecompilation();
- }
- else if (pos == 3){
- predHops = fsb.getIncrementHops();
- recompile = fsb.requiresIncrementRecompilation();
- }
- tmp = (IntObject) executePredicate(instructions, predHops, recompile, ValueType.INT, ec);
- }
- else
- tmp = (IntObject) executePredicate(instructions, null, false, ValueType.INT, ec);
- }
- }
- catch(Exception ex)
- {
- String predNameStr = null;
- if (pos == 1) predNameStr = "from";
- else if (pos == 2) predNameStr = "to";
- else if (pos == 3) predNameStr = "increment";
-
- throw new DMLRuntimeException(this.printBlockErrorLocation() +"Error evaluating '" + predNameStr + "' predicate", ex);
- }
-
- //final check of resulting int object (guaranteed to be non-null, see executePredicate)
- if( tmp instanceof IntObject )
- ret = (IntObject)tmp;
- else //downcast to int if necessary
- ret = new IntObject(tmp.getName(),tmp.getLongValue());
-
- return ret;
- }
-
- public String printBlockErrorLocation(){
- return "ERROR: Runtime error in for program block generated from for statement block between lines " + _beginLine + " and " + _endLine + " -- ";
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/controlprogram/FunctionProgramBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/FunctionProgramBlock.java b/src/main/java/com/ibm/bi/dml/runtime/controlprogram/FunctionProgramBlock.java
deleted file mode 100644
index 9785a38..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/FunctionProgramBlock.java
+++ /dev/null
@@ -1,166 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime.controlprogram;
-
-import java.util.ArrayList;
-
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.hops.recompile.Recompiler;
-import com.ibm.bi.dml.parser.DataIdentifier;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLScriptException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.instructions.cp.Data;
-import com.ibm.bi.dml.utils.Statistics;
-
-
-public class FunctionProgramBlock extends ProgramBlock
-{
-
- protected ArrayList<ProgramBlock> _childBlocks;
- protected ArrayList<DataIdentifier> _inputParams;
- protected ArrayList<DataIdentifier> _outputParams;
-
- private boolean _recompileOnce = false;
-
- public FunctionProgramBlock( Program prog, ArrayList<DataIdentifier> inputParams, ArrayList<DataIdentifier> outputParams) throws DMLRuntimeException
- {
- super(prog);
- _childBlocks = new ArrayList<ProgramBlock>();
- _inputParams = new ArrayList<DataIdentifier>();
- for (DataIdentifier id : inputParams){
- _inputParams.add(new DataIdentifier(id));
-
- }
- _outputParams = new ArrayList<DataIdentifier>();
- for (DataIdentifier id : outputParams){
- _outputParams.add(new DataIdentifier(id));
- }
- }
-
- public ArrayList<DataIdentifier> getInputParams(){
- return _inputParams;
- }
-
- public ArrayList<DataIdentifier> getOutputParams(){
- return _outputParams;
- }
-
- public void addProgramBlock(ProgramBlock childBlock) {
- _childBlocks.add(childBlock);
- }
-
- public void setChildBlocks( ArrayList<ProgramBlock> pbs)
- {
- _childBlocks = pbs;
- }
-
- public ArrayList<ProgramBlock> getChildBlocks() {
- return _childBlocks;
- }
-
- @Override
- public void execute(ExecutionContext ec)
- throws DMLRuntimeException, DMLUnsupportedOperationException
- {
- //dynamically recompile entire function body (according to function inputs)
- try {
- if( OptimizerUtils.ALLOW_DYN_RECOMPILATION
- && isRecompileOnce()
- && ParForProgramBlock.RESET_RECOMPILATION_FLAGs )
- {
- long t0 = DMLScript.STATISTICS ? System.nanoTime() : 0;
-
- //note: it is important to reset the recompilation flags here
- // (1) it is safe to reset recompilation flags because a 'recompile_once'
- // function will be recompiled for every execution.
- // (2) without reset, there would be no benefit in recompiling the entire function
- LocalVariableMap tmp = (LocalVariableMap) ec.getVariables().clone();
- Recompiler.recompileProgramBlockHierarchy(_childBlocks, tmp, _tid, true);
-
- if( DMLScript.STATISTICS ){
- long t1 = System.nanoTime();
- Statistics.incrementFunRecompileTime(t1-t0);
- Statistics.incrementFunRecompiles();
- }
- }
- }
- catch(Exception ex) {
- throw new DMLRuntimeException("Error recompiling function body.", ex);
- }
-
- // for each program block
- try {
- for (int i=0 ; i < this._childBlocks.size() ; i++) {
- ec.updateDebugState(i);
- _childBlocks.get(i).execute(ec);
- }
- }
- catch (DMLScriptException e) {
- throw e;
- }
- catch (Exception e){
- throw new DMLRuntimeException(this.printBlockErrorLocation() + "Error evaluating function program block", e);
- }
-
- // check return values
- checkOutputParameters(ec.getVariables());
- }
-
- /**
- *
- * @param vars
- */
- protected void checkOutputParameters( LocalVariableMap vars )
- {
- for( DataIdentifier diOut : _outputParams )
- {
- String varName = diOut.getName();
- Data dat = vars.get( varName );
- if( dat == null )
- LOG.error("Function output "+ varName +" is missing.");
- else if( dat.getDataType() != diOut.getDataType() )
- LOG.warn("Function output "+ varName +" has wrong data type: "+dat.getDataType()+".");
- else if( dat.getValueType() != diOut.getValueType() )
- LOG.warn("Function output "+ varName +" has wrong value type: "+dat.getValueType()+".");
-
- }
- }
-
- public void setRecompileOnce( boolean flag ) {
- _recompileOnce = flag;
- }
-
- public boolean isRecompileOnce() {
- return _recompileOnce;
- }
-
- public void printMe() {
-
- for (ProgramBlock pb : this._childBlocks){
- pb.printMe();
- }
- }
-
- public String printBlockErrorLocation(){
- return "ERROR: Runtime error in function program block generated from function statement block between lines " + _beginLine + " and " + _endLine + " -- ";
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/controlprogram/IfProgramBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/IfProgramBlock.java b/src/main/java/com/ibm/bi/dml/runtime/controlprogram/IfProgramBlock.java
deleted file mode 100644
index 925b426..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/IfProgramBlock.java
+++ /dev/null
@@ -1,276 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime.controlprogram;
-
-import java.util.ArrayList;
-
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.hops.Hop;
-import com.ibm.bi.dml.parser.IfStatementBlock;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLScriptException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.Instruction.INSTRUCTION_TYPE;
-import com.ibm.bi.dml.runtime.instructions.cp.BooleanObject;
-import com.ibm.bi.dml.runtime.instructions.cp.CPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.ComputationCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.Data;
-import com.ibm.bi.dml.runtime.instructions.cp.ScalarObject;
-import com.ibm.bi.dml.runtime.instructions.cp.StringObject;
-import com.ibm.bi.dml.runtime.instructions.cp.VariableCPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.CPInstruction.CPINSTRUCTION_TYPE;
-import com.ibm.bi.dml.yarn.DMLAppMasterUtils;
-
-
-public class IfProgramBlock extends ProgramBlock
-{
-
- private ArrayList<Instruction> _predicate;
- private String _predicateResultVar;
- private ArrayList <Instruction> _exitInstructions ;
- private ArrayList<ProgramBlock> _childBlocksIfBody;
- private ArrayList<ProgramBlock> _childBlocksElseBody;
-
- public IfProgramBlock(Program prog, ArrayList<Instruction> predicate) throws DMLRuntimeException{
- super(prog);
-
- _childBlocksIfBody = new ArrayList<ProgramBlock>();
- _childBlocksElseBody = new ArrayList<ProgramBlock>();
-
- _predicate = predicate;
- _predicateResultVar = findPredicateResultVar ();
- _exitInstructions = new ArrayList<Instruction>();
- }
-
- public ArrayList<ProgramBlock> getChildBlocksIfBody()
- { return _childBlocksIfBody; }
-
- public void setChildBlocksIfBody(ArrayList<ProgramBlock> blocks)
- { _childBlocksIfBody = blocks; }
-
- public void addProgramBlockIfBody(ProgramBlock pb)
- { _childBlocksIfBody.add(pb); }
-
- public ArrayList<ProgramBlock> getChildBlocksElseBody()
- { return _childBlocksElseBody; }
-
- public void setChildBlocksElseBody(ArrayList<ProgramBlock> blocks)
- { _childBlocksElseBody = blocks; }
-
- public void addProgramBlockElseBody(ProgramBlock pb)
- { _childBlocksElseBody.add(pb); }
-
- public void printMe() {
-
- LOG.debug("***** if current block predicate inst: *****");
- for (Instruction cp : _predicate){
- cp.printMe();
- }
-
- LOG.debug("***** children block inst --- if body : *****");
- for (ProgramBlock pb : this._childBlocksIfBody){
- pb.printMe();
- }
-
- LOG.debug("***** children block inst --- else body : *****");
- for (ProgramBlock pb: this._childBlocksElseBody){
- pb.printMe();
- }
-
- LOG.debug("***** current block inst exit: *****");
- for (Instruction i : this._exitInstructions) {
- i.printMe();
- }
- }
-
-
- public void setExitInstructions2(ArrayList<Instruction> exitInstructions){
- _exitInstructions = exitInstructions;
- }
-
- public void setExitInstructions1(ArrayList<Instruction> predicate){
- _predicate = predicate;
- }
-
- public void addExitInstruction(Instruction inst){
- _exitInstructions.add(inst);
- }
-
- public ArrayList<Instruction> getPredicate(){
- return _predicate;
- }
-
- public void setPredicate(ArrayList<Instruction> predicate)
- {
- _predicate = predicate;
-
- //update result var if non-empty predicate (otherwise,
- //do not overwrite varname predicate in predicateResultVar)
- if( _predicate != null && !_predicate.isEmpty() )
- _predicateResultVar = findPredicateResultVar();
- }
-
- public String getPredicateResultVar(){
- return _predicateResultVar;
- }
-
- public void setPredicateResultVar(String resultVar) {
- _predicateResultVar = resultVar;
- }
-
- public ArrayList<Instruction> getExitInstructions(){
- return _exitInstructions;
- }
-
- @Override
- public void execute(ExecutionContext ec)
- throws DMLRuntimeException, DMLUnsupportedOperationException
- {
- BooleanObject predResult = executePredicate(ec);
-
- //execute if statement
- if(predResult.getBooleanValue())
- {
- try
- {
- for (int i=0 ; i < _childBlocksIfBody.size() ; i++) {
- ec.updateDebugState(i);
- _childBlocksIfBody.get(i).execute(ec);
- }
- }
- catch(DMLScriptException e) {
- throw e;
- }
- catch(Exception e)
- {
- throw new DMLRuntimeException(this.printBlockErrorLocation() + "Error evaluating if statement body ", e);
- }
- }
- else
- {
- try
- {
- for (int i=0 ; i < _childBlocksElseBody.size() ; i++) {
- ec.updateDebugState(i);
- _childBlocksElseBody.get(i).execute(ec);
- }
- }
- catch(DMLScriptException e) {
- throw e;
- }
- catch(Exception e)
- {
- throw new DMLRuntimeException(this.printBlockErrorLocation() + "Error evaluating else statement body ", e);
- }
- }
-
- //execute exit instructions
- try {
- executeInstructions(_exitInstructions, ec);
- }
- catch(DMLScriptException e) {
- throw e;
- }
- catch (Exception e){
-
- throw new DMLRuntimeException(this.printBlockErrorLocation() + "Error evaluating if exit instructions ", e);
- }
- }
-
- /**
- *
- * @param ec
- * @return
- * @throws DMLRuntimeException
- * @throws DMLUnsupportedOperationException
- */
- private BooleanObject executePredicate(ExecutionContext ec)
- throws DMLRuntimeException, DMLUnsupportedOperationException
- {
- BooleanObject result = null;
- try
- {
- if( _predicate!=null && !_predicate.isEmpty() )
- {
- if( _sb != null )
- {
- if( DMLScript.isActiveAM() ) //set program block specific remote memory
- DMLAppMasterUtils.setupProgramBlockRemoteMaxMemory(this);
-
- IfStatementBlock isb = (IfStatementBlock)_sb;
- Hop predicateOp = isb.getPredicateHops();
- boolean recompile = isb.requiresPredicateRecompilation();
- result = (BooleanObject) executePredicate(_predicate, predicateOp, recompile, ValueType.BOOLEAN, ec);
- }
- else
- result = (BooleanObject) executePredicate(_predicate, null, false, ValueType.BOOLEAN, ec);
- }
- else
- {
- //get result var
- ScalarObject scalarResult = null;
- Data resultData = ec.getVariable(_predicateResultVar);
- if ( resultData == null ) {
- //note: resultvar is a literal (can it be of any value type other than String, hence no literal/varname conflict)
- scalarResult = ec.getScalarInput(_predicateResultVar, ValueType.BOOLEAN, true);
- }
- else {
- scalarResult = ec.getScalarInput(_predicateResultVar, ValueType.BOOLEAN, false);
- }
-
- //check for invalid type String
- if (scalarResult instanceof StringObject)
- throw new DMLRuntimeException(this.printBlockErrorLocation() + "\nIf predicate variable "+ _predicateResultVar + " evaluated to string " + scalarResult + " which is not allowed for predicates in DML");
-
- //process result
- if( scalarResult instanceof BooleanObject )
- result = (BooleanObject)scalarResult;
- else
- result = new BooleanObject( scalarResult.getBooleanValue() ); //auto casting
- }
- }
- catch(Exception ex)
- {
- throw new DMLRuntimeException(this.printBlockErrorLocation() + "Failed to evaluate the IF predicate.", ex);
- }
-
- //(guaranteed to be non-null, see executePredicate/getScalarInput)
- return result;
- }
-
- private String findPredicateResultVar ( ) {
- String result = null;
- for ( Instruction si : _predicate ) {
- if ( si.getType() == INSTRUCTION_TYPE.CONTROL_PROGRAM && ((CPInstruction)si).getCPInstructionType() != CPINSTRUCTION_TYPE.Variable ) {
- result = ((ComputationCPInstruction) si).getOutputVariableName();
- }
- else if(si instanceof VariableCPInstruction && ((VariableCPInstruction)si).isVariableCastInstruction()){
- result = ((VariableCPInstruction)si).getOutputVariableName();
- }
- }
- return result;
- }
-
- public String printBlockErrorLocation(){
- return "ERROR: Runtime error in if program block generated from if statement block between lines " + _beginLine + " and " + _endLine + " -- ";
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/runtime/controlprogram/LocalVariableMap.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/LocalVariableMap.java b/src/main/java/com/ibm/bi/dml/runtime/controlprogram/LocalVariableMap.java
deleted file mode 100644
index 5c5132d..0000000
--- a/src/main/java/com/ibm/bi/dml/runtime/controlprogram/LocalVariableMap.java
+++ /dev/null
@@ -1,231 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.runtime.controlprogram;
-
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.ProgramConverter;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.util.IDSequence;
-import com.ibm.bi.dml.runtime.instructions.cp.Data;
-import com.ibm.bi.dml.runtime.instructions.spark.data.LineageObject;
-
-import java.util.HashMap;
-import java.util.StringTokenizer;
-import java.util.Map.Entry;
-import java.util.Set;
-
-/**
- * Replaces <code>HashMap⟨String, Data⟩</code> as the table of
- * variable names and references. No longer supports global consistency.
- *
- */
-public class LocalVariableMap implements Cloneable
-{
- private static String eol = System.getProperty ("line.separator");
- private static String ELEMENT_DELIM = com.ibm.bi.dml.runtime.controlprogram.parfor.ProgramConverter.ELEMENT_DELIM;
- private static IDSequence _seq = new IDSequence();
-
- private HashMap <String, Data> localMap = null;
- private final long localID;
-
- public LocalVariableMap()
- {
- localMap = new HashMap <String, Data>();
- localID = _seq.getNextID();
- }
-
- public LocalVariableMap(LocalVariableMap vars)
- {
- localMap = new HashMap <String, Data>(vars.localMap);
- localID = _seq.getNextID();
- }
-
- /**
- *
- * @return
- */
- public Set<String> keySet()
- {
- return localMap.keySet();
- }
-
- /**
- * Retrieves the data object given its name.
- *
- * @param name : the variable name for the data object
- * @return the direct reference to the data object
- */
- public Data get( String name )
- {
- return localMap.get( name );
- }
-
- /**
- * Adds a new (name, value) pair to the variable map, or replaces an old pair with
- * the same name. Several different variable names may refer to the same value.
- *
- * @param name : the variable name for the data value
- * @param val : the data value object (such as envelope)
- */
- public void put(String name, Data val)
- {
- localMap.put( name, val );
- }
-
- /**
- *
- * @param vars
- */
- public void putAll( LocalVariableMap vars )
- {
- if( vars == this || vars == null )
- return;
- localMap.putAll (vars.localMap);
- }
-
- /**
- *
- * @param name
- */
- public Data remove( String name )
- {
- return localMap.remove( name );
- }
-
- /**
- *
- */
- public void removeAll()
- {
- localMap.clear();
- }
-
- /**
- *
- * @param d
- * @return
- */
- public boolean hasReferences( Data d )
- {
- return localMap.containsValue(d);
- }
-
- /**
- *
- * @param bo
- * @return
- */
- public boolean hasReferences( LineageObject bo )
- {
- for( Data tmpdat : localMap.values() )
- if ( tmpdat instanceof MatrixObject ) {
- MatrixObject mo = (MatrixObject)tmpdat;
- if( mo.getBroadcastHandle()==bo || mo.getRDDHandle()==bo )
- return true;
- }
- return false;
- }
-
- /**
- *
- * @param d
- * @param earlyAbort
- * @return
- */
- public int getNumReferences( Data d, boolean earlyAbort )
- {
- if ( d == null )
- return 0;
-
- int refCount = 0;
- for( Data tmpdat : localMap.values() )
- if ( tmpdat == d )
- if( ++refCount > 1 && earlyAbort )
- return refCount;
-
- return refCount;
- }
-
- /**
- *
- * @return
- * @throws DMLRuntimeException
- */
- public String serialize()
- throws DMLRuntimeException
- {
- StringBuilder sb = new StringBuilder();
-
- int count = 0;
- for (Entry <String, Data> e : localMap.entrySet ())
- {
- if (count != 0)
- sb.append (ELEMENT_DELIM);
- sb.append (ProgramConverter.serializeDataObject (e.getKey(), e.getValue()));
- count++;
- }
-
- return sb.toString();
- }
-
- /**
- *
- * @param varStr
- * @return
- * @throws DMLRuntimeException
- */
- public static LocalVariableMap deserialize(String varStr)
- throws DMLRuntimeException
- {
- StringTokenizer st2 = new StringTokenizer (varStr, ELEMENT_DELIM );
- LocalVariableMap vars = new LocalVariableMap ();
- while( st2.hasMoreTokens() )
- {
- String tmp = st2.nextToken().trim();
- Object[] tmp2 = ProgramConverter.parseDataObject (tmp);
- vars.put ((String) tmp2 [0], (Data) tmp2 [1]);
- }
- return vars;
- }
-
- @Override
- public String toString()
- {
- StringBuilder sb = new StringBuilder();
- sb.append("Local Variable Map ID = \"");
- sb.append(localID);
- sb.append("\":");
- sb.append(eol);
-
- for (Entry <String, Data> pair : localMap.entrySet()) {
- sb.append(" ");
- sb.append(pair.getKey());
- sb.append(" = ");
- sb.append(pair.getValue());
- sb.append(eol);
- }
-
- return sb.toString();
- }
-
- @Override
- public Object clone()
- {
- return new LocalVariableMap( this );
- }
-}