You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by lr...@apache.org on 2015/12/03 19:45:53 UTC
[22/78] [abbrv] [partial] incubator-systemml git commit: Move files
to new package folder structure
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/parser/BuiltinFunctionExpression.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/parser/BuiltinFunctionExpression.java b/src/main/java/com/ibm/bi/dml/parser/BuiltinFunctionExpression.java
deleted file mode 100644
index 2f39797..0000000
--- a/src/main/java/com/ibm/bi/dml/parser/BuiltinFunctionExpression.java
+++ /dev/null
@@ -1,1425 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.parser;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-
-import com.ibm.bi.dml.parser.LanguageException.LanguageErrorCodes;
-
-public class BuiltinFunctionExpression extends DataIdentifier
-{
-
- protected Expression[] _args = null;
- private BuiltinFunctionOp _opcode;
-
- public BuiltinFunctionExpression(BuiltinFunctionOp bifop, ArrayList<ParameterExpression> args, String fname, int blp, int bcp, int elp, int ecp) {
- _kind = Kind.BuiltinFunctionOp;
- _opcode = bifop;
- _args = new Expression[args.size()];
- for(int i=0; i < args.size(); i++) {
- _args[i] = args.get(i).getExpr();
- }
- this.setAllPositions(fname, blp, bcp, elp, ecp);
- }
-
- public BuiltinFunctionExpression(BuiltinFunctionOp bifop, Expression[] args, String fname, int blp, int bcp, int elp, int ecp) {
- _kind = Kind.BuiltinFunctionOp;
- _opcode = bifop;
- _args = new Expression[args.length];
- for(int i=0; i < args.length; i++) {
- _args[i] = args[i];
- }
- this.setAllPositions(fname, blp, bcp, elp, ecp);
- }
-
- public Expression rewriteExpression(String prefix) throws LanguageException {
-
- Expression[] newArgs = new Expression[_args.length];
- for(int i=0; i < _args.length; i++) {
- newArgs[i] = _args[i].rewriteExpression(prefix);
- }
- BuiltinFunctionExpression retVal = new BuiltinFunctionExpression(this._opcode, newArgs,
- this.getFilename(), this.getBeginLine(), this.getBeginColumn(), this.getEndLine(), this.getEndColumn());
- return retVal;
-
- }
-
- public BuiltinFunctionOp getOpCode() {
- return _opcode;
- }
-
- public Expression getFirstExpr() {
- return (_args.length >= 1 ? _args[0] : null);
- }
-
- public Expression getSecondExpr() {
- return (_args.length >= 2 ? _args[1] : null);
- }
-
- public Expression getThirdExpr() {
- return (_args.length >= 3 ? _args[2] : null);
- }
-
- public Expression[] getAllExpr(){
- return _args;
- }
-
- @Override
- public void validateExpression(MultiAssignmentStatement stmt, HashMap<String, DataIdentifier> ids, HashMap<String, ConstIdentifier> constVars, boolean conditional)
- throws LanguageException
- {
- if (this.getFirstExpr() instanceof FunctionCallIdentifier){
- raiseValidateError("UDF function call not supported as parameter to built-in function call", false);
- }
-
- this.getFirstExpr().validateExpression(ids, constVars, conditional);
- if (getSecondExpr() != null){
- if (this.getSecondExpr() instanceof FunctionCallIdentifier){
- raiseValidateError("UDF function call not supported as parameter to built-in function call", false);
- }
- getSecondExpr().validateExpression(ids, constVars, conditional);
- }
- if (getThirdExpr() != null) {
- if (this.getThirdExpr() instanceof FunctionCallIdentifier){
- raiseValidateError("UDF function call not supported as parameter to built-in function call", false);
- }
- getThirdExpr().validateExpression(ids, constVars, conditional);
- }
- _outputs = new Identifier[stmt.getTargetList().size()];
- int count = 0;
- for (DataIdentifier outParam: stmt.getTargetList()){
- DataIdentifier tmp = new DataIdentifier(outParam);
- tmp.setAllPositions(this.getFilename(), this.getBeginLine(), this.getBeginColumn(), this.getEndLine(), this.getEndColumn());
- _outputs[count++] = tmp;
- }
-
- switch (_opcode) {
- case QR:
- checkNumParameters(1);
- checkMatrixParam(getFirstExpr());
-
- // setup output properties
- DataIdentifier qrOut1 = (DataIdentifier) getOutputs()[0];
- DataIdentifier qrOut2 = (DataIdentifier) getOutputs()[1];
-
- long rows = getFirstExpr().getOutput().getDim1();
- long cols = getFirstExpr().getOutput().getDim2();
-
- // Output1 - Q
- qrOut1.setDataType(DataType.MATRIX);
- qrOut1.setValueType(ValueType.DOUBLE);
- qrOut1.setDimensions(rows, cols);
- qrOut1.setBlockDimensions(getFirstExpr().getOutput().getRowsInBlock(), getFirstExpr().getOutput().getColumnsInBlock());
-
- // Output2 - R
- qrOut2.setDataType(DataType.MATRIX);
- qrOut2.setValueType(ValueType.DOUBLE);
- qrOut2.setDimensions(rows, cols);
- qrOut2.setBlockDimensions(getFirstExpr().getOutput().getRowsInBlock(), getFirstExpr().getOutput().getColumnsInBlock());
-
- break;
-
- case LU:
- checkNumParameters(1);
- checkMatrixParam(getFirstExpr());
-
- // setup output properties
- DataIdentifier luOut1 = (DataIdentifier) getOutputs()[0];
- DataIdentifier luOut2 = (DataIdentifier) getOutputs()[1];
- DataIdentifier luOut3 = (DataIdentifier) getOutputs()[2];
-
- long inrows = getFirstExpr().getOutput().getDim1();
- long incols = getFirstExpr().getOutput().getDim2();
-
- if ( inrows != incols ) {
- raiseValidateError("LU Decomposition can only be done on a square matrix. Input matrix is rectangular (rows=" + inrows + ", cols="+incols+")", conditional);
- }
-
- // Output1 - P
- luOut1.setDataType(DataType.MATRIX);
- luOut1.setValueType(ValueType.DOUBLE);
- luOut1.setDimensions(inrows, inrows);
- luOut1.setBlockDimensions(getFirstExpr().getOutput().getRowsInBlock(), getFirstExpr().getOutput().getColumnsInBlock());
-
- // Output2 - L
- luOut2.setDataType(DataType.MATRIX);
- luOut2.setValueType(ValueType.DOUBLE);
- luOut2.setDimensions(inrows, inrows);
- luOut2.setBlockDimensions(getFirstExpr().getOutput().getRowsInBlock(), getFirstExpr().getOutput().getColumnsInBlock());
-
- // Output3 - U
- luOut3.setDataType(DataType.MATRIX);
- luOut3.setValueType(ValueType.DOUBLE);
- luOut3.setDimensions(inrows, inrows);
- luOut3.setBlockDimensions(getFirstExpr().getOutput().getRowsInBlock(), getFirstExpr().getOutput().getColumnsInBlock());
-
- break;
-
- case EIGEN:
- checkNumParameters(1);
- checkMatrixParam(getFirstExpr());
-
- // setup output properties
- DataIdentifier eigenOut1 = (DataIdentifier) getOutputs()[0];
- DataIdentifier eigenOut2 = (DataIdentifier) getOutputs()[1];
-
- if ( getFirstExpr().getOutput().getDim1() != getFirstExpr().getOutput().getDim2() ) {
- raiseValidateError("Eigen Decomposition can only be done on a square matrix. Input matrix is rectangular (rows=" + getFirstExpr().getOutput().getDim1() + ", cols="+ getFirstExpr().getOutput().getDim2() +")", conditional);
- }
-
- // Output1 - Eigen Values
- eigenOut1.setDataType(DataType.MATRIX);
- eigenOut1.setValueType(ValueType.DOUBLE);
- eigenOut1.setDimensions(getFirstExpr().getOutput().getDim1(), 1);
- eigenOut1.setBlockDimensions(getFirstExpr().getOutput().getRowsInBlock(), getFirstExpr().getOutput().getColumnsInBlock());
-
- // Output2 - Eigen Vectors
- eigenOut2.setDataType(DataType.MATRIX);
- eigenOut2.setValueType(ValueType.DOUBLE);
- eigenOut2.setDimensions(getFirstExpr().getOutput().getDim1(), getFirstExpr().getOutput().getDim2());
- eigenOut2.setBlockDimensions(getFirstExpr().getOutput().getRowsInBlock(), getFirstExpr().getOutput().getColumnsInBlock());
-
- break;
-
- default: //always unconditional
- raiseValidateError("Unknown Builtin Function opcode: " + _opcode, false);
- }
- }
-
- /**
- * Validate parse tree : Process BuiltinFunction Expression in an assignment
- * statement
- *
- * @throws LanguageException
- */
- public void validateExpression(HashMap<String, DataIdentifier> ids, HashMap<String, ConstIdentifier> constVars, boolean conditional)
- throws LanguageException {
-
- for(int i=0; i < _args.length; i++ ) {
-
- if (_args[i] instanceof FunctionCallIdentifier){
- raiseValidateError("UDF function call not supported as parameter to built-in function call", false);
- }
-
- _args[i].validateExpression(ids, constVars, conditional);
- }
-
- // checkIdentifierParams();
- String outputName = getTempName();
- DataIdentifier output = new DataIdentifier(outputName);
- output.setAllPositions(this.getFilename(), this.getBeginLine(), this.getBeginColumn(), this.getEndLine(), this.getEndColumn());
-
- Identifier id = this.getFirstExpr().getOutput();
- output.setProperties(this.getFirstExpr().getOutput());
- output.setNnz(-1); //conservatively, cannot use input nnz!
- this.setOutput(output);
-
- switch (this.getOpCode()) {
- case COLSUM:
- case COLMAX:
- case COLMIN:
- case COLMEAN:
- // colSums(X);
- checkNumParameters(1);
- checkMatrixParam(getFirstExpr());
- output.setDataType(DataType.MATRIX);
- output.setDimensions(1, id.getDim2());
- output.setBlockDimensions (id.getRowsInBlock(), id.getColumnsInBlock());
- output.setValueType(id.getValueType());
- break;
- case ROWSUM:
- case ROWMAX:
- case ROWINDEXMAX:
- case ROWMIN:
- case ROWINDEXMIN:
- case ROWMEAN:
- //rowSums(X);
- checkNumParameters(1);
- checkMatrixParam(getFirstExpr());
- output.setDataType(DataType.MATRIX);
- output.setDimensions(id.getDim1(), 1);
- output.setBlockDimensions (id.getRowsInBlock(), id.getColumnsInBlock());
- output.setValueType(id.getValueType());
- break;
- case SUM:
- case PROD:
- case TRACE:
- // sum(X);
- checkNumParameters(1);
- checkMatrixParam(getFirstExpr());
-
- output.setDataType(DataType.SCALAR);
- output.setDimensions(0, 0);
- output.setBlockDimensions (0, 0);
- output.setValueType(id.getValueType());
-
- break;
-
- case MEAN:
- //checkNumParameters(2, false); // mean(Y) or mean(Y,W)
- if (getSecondExpr() != null) {
- checkNumParameters (2);
- }
- else {
- checkNumParameters (1);
- }
-
- checkMatrixParam(getFirstExpr());
- if ( getSecondExpr() != null ) {
- // x = mean(Y,W);
- checkMatchingDimensions(getFirstExpr(), getSecondExpr());
- }
-
- output.setDataType(DataType.SCALAR);
- output.setDimensions(0, 0);
- output.setBlockDimensions (0, 0);
- output.setValueType(id.getValueType());
- break;
-
- case MIN:
- case MAX:
- //min(X), min(X,s), min(s,X), min(s,r), min(X,Y)
-
- //unary aggregate
- if (getSecondExpr() == null)
- {
- checkNumParameters(1);
- checkMatrixParam(getFirstExpr());
- output.setDataType( DataType.SCALAR );
- output.setDimensions(0, 0);
- output.setBlockDimensions (0, 0);
- }
- //binary operation
- else
- {
- checkNumParameters(2);
- DataType dt1 = getFirstExpr().getOutput().getDataType();
- DataType dt2 = getSecondExpr().getOutput().getDataType();
- DataType dtOut = (dt1==DataType.MATRIX || dt2==DataType.MATRIX)?
- DataType.MATRIX : DataType.SCALAR;
- if( dt1==DataType.MATRIX && dt2==DataType.MATRIX )
- checkMatchingDimensions(getFirstExpr(), getSecondExpr(), true);
- //determine output dimensions
- long[] dims = getBinaryMatrixCharacteristics(getFirstExpr(), getSecondExpr());
- output.setDataType( dtOut );
- output.setDimensions(dims[0], dims[1]);
- output.setBlockDimensions (dims[2], dims[3]);
- }
- output.setValueType(id.getValueType());
-
- break;
-
- case CUMSUM:
- case CUMPROD:
- case CUMMIN:
- case CUMMAX:
- // cumsum(X);
- checkNumParameters(1);
- checkMatrixParam(getFirstExpr());
-
- output.setDataType(DataType.MATRIX);
- output.setDimensions(id.getDim1(), id.getDim2());
- output.setBlockDimensions (id.getRowsInBlock(), id.getColumnsInBlock());
- output.setValueType(id.getValueType());
-
- break;
-
- case CAST_AS_SCALAR:
- checkNumParameters(1);
- checkMatrixParam(getFirstExpr());
- if (( getFirstExpr().getOutput().getDim1() != -1 && getFirstExpr().getOutput().getDim1() !=1) || ( getFirstExpr().getOutput().getDim2() != -1 && getFirstExpr().getOutput().getDim2() !=1)) {
- raiseValidateError("dimension mismatch while casting matrix to scalar: dim1: " + getFirstExpr().getOutput().getDim1() + " dim2 " + getFirstExpr().getOutput().getDim2(),
- conditional, LanguageErrorCodes.INVALID_PARAMETERS);
- }
- output.setDataType(DataType.SCALAR);
- output.setDimensions(0, 0);
- output.setBlockDimensions (0, 0);
- output.setValueType(id.getValueType());
- break;
- case CAST_AS_MATRIX:
- checkNumParameters(1);
- checkScalarParam(getFirstExpr());
- output.setDataType(DataType.MATRIX);
- output.setDimensions(1, 1);
- output.setBlockDimensions(id.getRowsInBlock(), id.getColumnsInBlock());
- output.setValueType(id.getValueType());
- break;
- case CAST_AS_DOUBLE:
- checkNumParameters(1);
- checkScalarParam(getFirstExpr());
- output.setDataType(DataType.SCALAR);
- //output.setDataType(id.getDataType()); //TODO whenever we support multiple matrix value types, currently noop.
- output.setDimensions(0, 0);
- output.setBlockDimensions (0, 0);
- output.setValueType(ValueType.DOUBLE);
- break;
- case CAST_AS_INT:
- checkNumParameters(1);
- checkScalarParam(getFirstExpr());
- output.setDataType(DataType.SCALAR);
- //output.setDataType(id.getDataType()); //TODO whenever we support multiple matrix value types, currently noop.
- output.setDimensions(0, 0);
- output.setBlockDimensions (0, 0);
- output.setValueType(ValueType.INT);
- break;
- case CAST_AS_BOOLEAN:
- checkNumParameters(1);
- checkScalarParam(getFirstExpr());
- output.setDataType(DataType.SCALAR);
- //output.setDataType(id.getDataType()); //TODO whenever we support multiple matrix value types, currently noop.
- output.setDimensions(0, 0);
- output.setBlockDimensions (0, 0);
- output.setValueType(ValueType.BOOLEAN);
- break;
-
- case CBIND:
- case RBIND:
- checkNumParameters(2);
-
- //scalar string append (string concatenation with \n)
- if( getFirstExpr().getOutput().getDataType()==DataType.SCALAR ) {
- checkScalarParam(getFirstExpr());
- checkScalarParam(getSecondExpr());
- checkValueTypeParam(getFirstExpr(), ValueType.STRING);
- checkValueTypeParam(getSecondExpr(), ValueType.STRING);
- }
- //matrix append (rbind/cbind)
- else {
- checkMatrixParam(getFirstExpr());
- checkMatrixParam(getSecondExpr());
- }
-
- output.setDataType(id.getDataType());
- output.setValueType(id.getValueType());
-
- // set output dimensions and validate consistency
- long appendDim1 = -1, appendDim2 = -1;
- long m1rlen = getFirstExpr().getOutput().getDim1();
- long m1clen = getFirstExpr().getOutput().getDim2();
- long m2rlen = getSecondExpr().getOutput().getDim1();
- long m2clen = getSecondExpr().getOutput().getDim2();
-
- if( getOpCode() == BuiltinFunctionOp.CBIND ) {
- if (m1rlen > 0 && m2rlen > 0 && m1rlen!=m2rlen) {
- raiseValidateError("inputs to cbind must have same number of rows: input 1 rows: " +
- m1rlen+", input 2 rows: "+m2rlen, conditional, LanguageErrorCodes.INVALID_PARAMETERS);
- }
- appendDim1 = (m1rlen>0) ? m1rlen : m2rlen;
- appendDim2 = (m1clen>0 && m2clen>0)? m1clen + m2clen : -1;
- }
- else if( getOpCode() == BuiltinFunctionOp.RBIND ) {
- if (m1clen > 0 && m2clen > 0 && m1clen!=m2clen) {
- raiseValidateError("inputs to rbind must have same number of columns: input 1 columns: " +
- m1clen+", input 2 columns: "+m2clen, conditional, LanguageErrorCodes.INVALID_PARAMETERS);
- }
- appendDim1 = (m1rlen>0 && m2rlen>0)? m1rlen + m2rlen : -1;
- appendDim2 = (m1clen>0) ? m1clen : m2clen;
- }
-
- output.setDimensions(appendDim1, appendDim2);
- output.setBlockDimensions (id.getRowsInBlock(), id.getColumnsInBlock());
-
- break;
-
- case PPRED:
- // ppred (X,Y, "<"); ppred (X,y, "<"); ppred (y,X, "<");
- checkNumParameters(3);
-
- DataType dt1 = getFirstExpr().getOutput().getDataType();
- DataType dt2 = getSecondExpr().getOutput().getDataType();
-
- //check input data types
- if( dt1 == DataType.SCALAR && dt2 == DataType.SCALAR ) {
- raiseValidateError("ppred() requires at least one matrix input.", conditional, LanguageErrorCodes.INVALID_PARAMETERS);
- }
- if( dt1 == DataType.MATRIX )
- checkMatrixParam(getFirstExpr());
- if( dt2 == DataType.MATRIX )
- checkMatrixParam(getSecondExpr());
- if( dt1==DataType.MATRIX && dt2==DataType.MATRIX ) //dt1==dt2
- checkMatchingDimensions(getFirstExpr(), getSecondExpr(), true);
-
- //check operator
- if (getThirdExpr().getOutput().getDataType() != DataType.SCALAR ||
- getThirdExpr().getOutput().getValueType() != ValueType.STRING)
- {
- raiseValidateError("Third argument in ppred() is not an operator ", conditional, LanguageErrorCodes.INVALID_PARAMETERS);
- }
-
- //determine output dimensions
- long[] dims = getBinaryMatrixCharacteristics(getFirstExpr(), getSecondExpr());
- output.setDataType(DataType.MATRIX);
- output.setDimensions(dims[0], dims[1]);
- output.setBlockDimensions(dims[2], dims[3]);
- output.setValueType(id.getValueType());
- break;
-
- case TRANS:
- checkNumParameters(1);
- checkMatrixParam(getFirstExpr());
- output.setDataType(DataType.MATRIX);
- output.setDimensions(id.getDim2(), id.getDim1());
- output.setBlockDimensions (id.getColumnsInBlock(), id.getRowsInBlock());
- output.setValueType(id.getValueType());
- break;
- case DIAG:
- checkNumParameters(1);
- checkMatrixParam(getFirstExpr());
- output.setDataType(DataType.MATRIX);
- if( id.getDim2() != -1 ) { //type known
- if ( id.getDim2() == 1 )
- {
- //diag V2M
- output.setDimensions(id.getDim1(), id.getDim1());
- }
- else
- {
- if (id.getDim1() != id.getDim2()) {
- raiseValidateError("Invoking diag on matrix with dimensions ("
- + id.getDim1() + "," + id.getDim2()
- + ") in " + this.toString(), conditional, LanguageErrorCodes.INVALID_PARAMETERS);
- }
- //diag M2V
- output.setDimensions(id.getDim1(), 1);
- }
- }
- output.setBlockDimensions (id.getRowsInBlock(), id.getColumnsInBlock());
- output.setValueType(id.getValueType());
- break;
- case NROW:
- case NCOL:
- case LENGTH:
- checkNumParameters(1);
- checkMatrixParam(getFirstExpr());
- output.setDataType(DataType.SCALAR);
- output.setDimensions(0, 0);
- output.setBlockDimensions (0, 0);
- output.setValueType(ValueType.INT);
- break;
-
- // Contingency tables
- case TABLE:
-
- /*
- * Allowed #of arguments: 2,3,4,5
- * table(A,B)
- * table(A,B,W)
- * table(A,B,1)
- * table(A,B,dim1,dim2)
- * table(A,B,W,dim1,dim2)
- * table(A,B,1,dim1,dim2)
- */
-
- // Check for validity of input arguments, and setup output dimensions
-
- // First input: is always of type MATRIX
- checkMatrixParam(getFirstExpr());
-
- if ( getSecondExpr() == null )
- raiseValidateError("Invalid number of arguments to table(): "
- + this.toString(), conditional, LanguageErrorCodes.INVALID_PARAMETERS);
-
- // Second input: can be MATRIX or SCALAR
- // cases: table(A,B) or table(A,1)
- if ( getSecondExpr().getOutput().getDataType() == DataType.MATRIX)
- checkMatchingDimensions(getFirstExpr(),getSecondExpr());
-
- long outputDim1=-1, outputDim2=-1;
-
- switch(_args.length) {
- case 2:
- // nothing to do
- break;
-
- case 3:
- // case - table w/ weights
- // - weights specified as a matrix: table(A,B,W) or table(A,1,W)
- // - weights specified as a scalar: table(A,B,1) or table(A,1,1)
- if ( getThirdExpr().getOutput().getDataType() == DataType.MATRIX)
- checkMatchingDimensions(getFirstExpr(),getThirdExpr());
- break;
-
- case 4:
- // case - table w/ output dimensions: table(A,B,dim1,dim2) or table(A,1,dim1,dim2)
- // third and fourth arguments must be scalars
- if ( getThirdExpr().getOutput().getDataType() != DataType.SCALAR || _args[3].getOutput().getDataType() != DataType.SCALAR ) {
- raiseValidateError("Invalid argument types to table(): output dimensions must be of type scalar: "
- + this.toString(), conditional, LanguageErrorCodes.INVALID_PARAMETERS);
- }
- else {
- // constant propagation
- if( getThirdExpr() instanceof DataIdentifier && constVars.containsKey(((DataIdentifier)getThirdExpr()).getName()) )
- _args[2] = constVars.get(((DataIdentifier)getThirdExpr()).getName());
- if( _args[3] instanceof DataIdentifier && constVars.containsKey(((DataIdentifier)_args[3]).getName()) )
- _args[3] = constVars.get(((DataIdentifier)_args[3]).getName());
-
- if ( getThirdExpr().getOutput() instanceof ConstIdentifier )
- outputDim1 = ((ConstIdentifier) getThirdExpr().getOutput()).getLongValue();
- if ( _args[3].getOutput() instanceof ConstIdentifier )
- outputDim2 = ((ConstIdentifier) _args[3].getOutput()).getLongValue();
- }
- break;
-
- case 5:
- // case - table w/ weights and output dimensions:
- // - table(A,B,W,dim1,dim2) or table(A,1,W,dim1,dim2)
- // - table(A,B,1,dim1,dim2) or table(A,1,1,dim1,dim2)
-
- if ( getThirdExpr().getOutput().getDataType() == DataType.MATRIX)
- checkMatchingDimensions(getFirstExpr(),getThirdExpr());
-
- // fourth and fifth arguments must be scalars
- if ( _args[3].getOutput().getDataType() != DataType.SCALAR || _args[4].getOutput().getDataType() != DataType.SCALAR ) {
- raiseValidateError("Invalid argument types to table(): output dimensions must be of type scalar: "
- + this.toString(), conditional, LanguageErrorCodes.INVALID_PARAMETERS);
- }
- else {
- // constant propagation
- if( _args[3] instanceof DataIdentifier && constVars.containsKey(((DataIdentifier)_args[3]).getName()) )
- _args[3] = constVars.get(((DataIdentifier)_args[3]).getName());
- if( _args[4] instanceof DataIdentifier && constVars.containsKey(((DataIdentifier)_args[4]).getName()) )
- _args[4] = constVars.get(((DataIdentifier)_args[4]).getName());
-
- if ( _args[3].getOutput() instanceof ConstIdentifier )
- outputDim1 = ((ConstIdentifier) _args[3].getOutput()).getLongValue();
- if ( _args[4].getOutput() instanceof ConstIdentifier )
- outputDim2 = ((ConstIdentifier) _args[4].getOutput()).getLongValue();
- }
- break;
-
- default:
- raiseValidateError("Invalid number of arguments to table(): "
- + this.toString(), conditional, LanguageErrorCodes.INVALID_PARAMETERS);
- }
- // The dimensions for the output matrix will be known only at the
- // run time
- output.setDimensions(outputDim1, outputDim2);
- output.setBlockDimensions (-1, -1);
- output.setDataType(DataType.MATRIX);
- output.setValueType(ValueType.DOUBLE);
- break;
-
- case MOMENT:
- /*
- * x = centralMoment(V,order) or xw = centralMoment(V,W,order)
- */
- checkMatrixParam(getFirstExpr());
- if (getThirdExpr() != null) {
- checkNumParameters(3);
- checkMatrixParam(getSecondExpr());
- checkMatchingDimensions(getFirstExpr(),getSecondExpr());
- checkScalarParam(getThirdExpr());
- }
- else {
- checkNumParameters(2);
- checkScalarParam(getSecondExpr());
- }
-
- // output is a scalar
- output.setDataType(DataType.SCALAR);
- output.setValueType(ValueType.DOUBLE);
- output.setDimensions(0, 0);
- output.setBlockDimensions(0,0);
- break;
-
- case COV:
- /*
- * x = cov(V1,V2) or xw = cov(V1,V2,W)
- */
- if (getThirdExpr() != null) {
- checkNumParameters(3);
- }
- else {
- checkNumParameters(2);
- }
- checkMatrixParam(getFirstExpr());
- checkMatrixParam(getSecondExpr());
- checkMatchingDimensions(getFirstExpr(),getSecondExpr());
-
- if (getThirdExpr() != null) {
- checkMatrixParam(getThirdExpr());
- checkMatchingDimensions(getFirstExpr(), getThirdExpr());
- }
-
- // output is a scalar
- output.setDataType(DataType.SCALAR);
- output.setValueType(ValueType.DOUBLE);
- output.setDimensions(0, 0);
- output.setBlockDimensions(0,0);
- break;
-
- case QUANTILE:
- /*
- * q = quantile(V1,0.5) computes median in V1
- * or Q = quantile(V1,P) computes the vector of quantiles as specified by P
- * or qw = quantile(V1,W,0.5) computes median when weights (W) are given
- * or QW = quantile(V1,W,P) computes the vector of quantiles as specified by P, when weights (W) are given
- */
- if(getThirdExpr() != null) {
- checkNumParameters(3);
- }
- else {
- checkNumParameters(2);
- }
-
- // first parameter must always be a 1D matrix
- check1DMatrixParam(getFirstExpr());
-
- // check for matching dimensions for other matrix parameters
- if (getThirdExpr() != null) {
- checkMatrixParam(getSecondExpr());
- checkMatchingDimensions(getFirstExpr(), getSecondExpr());
- }
-
- // set the properties for _output expression
- // output dimensions = dimensions of second, if third is null
- // = dimensions of the third, otherwise.
-
- if (getThirdExpr() != null) {
- output.setDimensions(getThirdExpr().getOutput().getDim1(), getThirdExpr().getOutput()
- .getDim2());
- output.setBlockDimensions(getThirdExpr().getOutput().getRowsInBlock(),
- getThirdExpr().getOutput().getColumnsInBlock());
- output.setDataType(getThirdExpr().getOutput().getDataType());
- } else {
- output.setDimensions(getSecondExpr().getOutput().getDim1(), getSecondExpr().getOutput()
- .getDim2());
- output.setBlockDimensions(getSecondExpr().getOutput().getRowsInBlock(),
- getSecondExpr().getOutput().getColumnsInBlock());
- output.setDataType(getSecondExpr().getOutput().getDataType());
- }
- break;
-
- case INTERQUANTILE:
- if (getThirdExpr() != null) {
- checkNumParameters(3);
- }
- else {
- checkNumParameters(2);
- }
- checkMatrixParam(getFirstExpr());
- if (getThirdExpr() != null) {
- // i.e., second input is weight vector
- checkMatrixParam(getSecondExpr());
- checkMatchingDimensionsQuantile();
- }
-
- if ((getThirdExpr() == null && getSecondExpr().getOutput().getDataType() != DataType.SCALAR)
- && (getThirdExpr() != null && getThirdExpr().getOutput().getDataType() != DataType.SCALAR)) {
-
- raiseValidateError("Invalid parameters to "+ this.getOpCode(), conditional, LanguageErrorCodes.INVALID_PARAMETERS);
- }
-
- output.setValueType(id.getValueType());
- // output dimensions are unknown
- output.setDimensions(-1, -1);
- output.setBlockDimensions(-1,-1);
- output.setDataType(DataType.MATRIX);
- break;
-
- case IQM:
- /*
- * Usage: iqm = InterQuartileMean(A,W); iqm = InterQuartileMean(A);
- */
- if (getSecondExpr() != null){
- checkNumParameters(2);
- }
- else {
- checkNumParameters(1);
- }
- checkMatrixParam(getFirstExpr());
-
- if (getSecondExpr() != null) {
- // i.e., second input is weight vector
- checkMatrixParam(getSecondExpr());
- checkMatchingDimensions(getFirstExpr(), getSecondExpr());
- }
-
- // Output is a scalar
- output.setValueType(id.getValueType());
- output.setDimensions(0, 0);
- output.setBlockDimensions(0,0);
- output.setDataType(DataType.SCALAR);
-
- break;
-
- case MEDIAN:
- if (getSecondExpr() != null){
- checkNumParameters(2);
- }
- else {
- checkNumParameters(1);
- }
- checkMatrixParam(getFirstExpr());
-
- if (getSecondExpr() != null) {
- // i.e., second input is weight vector
- checkMatrixParam(getSecondExpr());
- checkMatchingDimensions(getFirstExpr(), getSecondExpr());
- }
-
- // Output is a scalar
- output.setValueType(id.getValueType());
- output.setDimensions(0, 0);
- output.setBlockDimensions(0,0);
- output.setDataType(DataType.SCALAR);
-
- break;
-
- case SAMPLE:
- {
- Expression[] in = getAllExpr();
-
- for(Expression e : in)
- checkScalarParam(e);
-
- if (in[0].getOutput().getValueType() != ValueType.DOUBLE && in[0].getOutput().getValueType() != ValueType.INT)
- throw new LanguageException("First argument to sample() must be a number.");
- if (in[1].getOutput().getValueType() != ValueType.DOUBLE && in[1].getOutput().getValueType() != ValueType.INT)
- throw new LanguageException("Second argument to sample() must be a number.");
-
- boolean check = false;
- if ( isConstant(in[0]) && isConstant(in[1]) )
- {
- long range = ((ConstIdentifier)in[0]).getLongValue();
- long size = ((ConstIdentifier)in[1]).getLongValue();
- if ( range < size )
- check = true;
- }
-
- if(in.length == 4 )
- {
- checkNumParameters(4);
- if (in[3].getOutput().getValueType() != ValueType.INT)
- throw new LanguageException("Fourth arugment, seed, to sample() must be an integer value.");
- if (in[2].getOutput().getValueType() != ValueType.BOOLEAN )
- throw new LanguageException("Third arugment to sample() must either denote replacement policy (boolean) or seed (integer).");
- }
- else if(in.length == 3)
- {
- checkNumParameters(3);
- if (in[2].getOutput().getValueType() != ValueType.BOOLEAN
- && in[2].getOutput().getValueType() != ValueType.INT )
- throw new LanguageException("Third arugment to sample() must either denote replacement policy (boolean) or seed (integer).");
- }
-
- if ( check && in.length >= 3
- && isConstant(in[2])
- && in[2].getOutput().getValueType() == ValueType.BOOLEAN
- && !((BooleanIdentifier)in[2]).getValue() )
- throw new LanguageException("Sample (size=" + ((ConstIdentifier)in[0]).getLongValue()
- + ") larger than population (size=" + ((ConstIdentifier)in[1]).getLongValue()
- + ") can only be generated with replacement.");
-
- // Output is a column vector
- output.setDataType(DataType.MATRIX);
- output.setValueType(ValueType.DOUBLE);
-
- if ( isConstant(in[1]) )
- output.setDimensions(((ConstIdentifier)in[1]).getLongValue(), 1);
- else
- output.setDimensions(-1, 1);
- setBlockDimensions(id.getRowsInBlock(), id.getColumnsInBlock());
-
- break;
- }
- case SEQ:
-
- //basic parameter validation
- checkScalarParam(getFirstExpr());
- checkScalarParam(getSecondExpr());
- if ( getThirdExpr() != null ) {
- checkNumParameters(3);
- checkScalarParam(getThirdExpr());
- }
- else
- checkNumParameters(2);
-
- // constant propagation (from, to, incr)
- if( getFirstExpr() instanceof DataIdentifier && constVars.containsKey(((DataIdentifier)getFirstExpr()).getName()) )
- _args[0] = constVars.get(((DataIdentifier)getFirstExpr()).getName());
- if( getSecondExpr() instanceof DataIdentifier && constVars.containsKey(((DataIdentifier)getSecondExpr()).getName()) )
- _args[1] = constVars.get(((DataIdentifier)getSecondExpr()).getName());
- if( getThirdExpr()!=null && getThirdExpr() instanceof DataIdentifier && constVars.containsKey(((DataIdentifier)getThirdExpr()).getName()) )
- _args[2] = constVars.get(((DataIdentifier)getThirdExpr()).getName());
-
- // check if dimensions can be inferred
- long dim1=-1, dim2=1;
- if ( isConstant(getFirstExpr()) && isConstant(getSecondExpr()) && (getThirdExpr() != null ? isConstant(getThirdExpr()) : true) ) {
- double from, to, incr;
- boolean neg;
- try {
- from = getDoubleValue(getFirstExpr());
- to = getDoubleValue(getSecondExpr());
-
- // Setup the value of increment
- // default value: 1 if from <= to; -1 if from > to
- neg = (from > to);
- if(getThirdExpr() == null) {
- expandArguments();
- _args[2] = new DoubleIdentifier((neg? -1.0 : 1.0),
- this.getFilename(), this.getBeginLine(), this.getBeginColumn(),
- this.getEndLine(), this.getEndColumn());
- }
- incr = getDoubleValue(getThirdExpr());
-
- }
- catch (LanguageException e) {
- throw new LanguageException("Arguments for seq() must be numeric.");
- }
-
- if (neg != (incr < 0))
- throw new LanguageException("Wrong sign for the increment in a call to seq()");
-
- // Both end points of the range must included i.e., [from,to] both inclusive.
- // Note that, "to" is included only if (to-from) is perfectly divisible by incr
- // For example, seq(0,1,0.5) produces (0.0 0.5 1.0) whereas seq(0,1,0.6) produces only (0.0 0.6) but not (0.0 0.6 1.0)
- dim1 = 1 + (long)Math.floor((to-from)/incr);
- //System.out.println("seq("+from+","+to+","+incr+") -> dims("+dim1+","+dim2+")");
- }
- output.setDataType(DataType.MATRIX);
- output.setValueType(ValueType.DOUBLE);
- output.setDimensions(dim1, dim2);
- output.setBlockDimensions(0, 0);
- break;
-
- case SOLVE:
- checkNumParameters(2);
- checkMatrixParam(getFirstExpr());
- checkMatrixParam(getSecondExpr());
-
- if ( getSecondExpr().getOutput().dimsKnown() && !is1DMatrix(getSecondExpr()) )
- raiseValidateError("Second input to solve() must be a vector", conditional);
-
- if ( getFirstExpr().getOutput().dimsKnown() && getSecondExpr().getOutput().dimsKnown() &&
- getFirstExpr().getOutput().getDim1() != getSecondExpr().getOutput().getDim1() )
- raiseValidateError("Dimension mismatch in a call to solve()", conditional);
-
- output.setDataType(DataType.MATRIX);
- output.setValueType(ValueType.DOUBLE);
- output.setDimensions(getFirstExpr().getOutput().getDim2(), 1);
- output.setBlockDimensions(0, 0);
- break;
-
- case INVERSE:
- checkNumParameters(1);
- checkMatrixParam(getFirstExpr());
-
- output.setDataType(DataType.MATRIX);
- output.setValueType(ValueType.DOUBLE);
-
- Identifier in = getFirstExpr().getOutput();
- if(in.dimsKnown() && in.getDim1() != in.getDim2())
- raiseValidateError("Input to inv() must be square matrix -- given: a " + in.getDim1() + "x" + in.getDim2() + " matrix.", conditional);
-
- output.setDimensions(in.getDim1(), in.getDim2());
- output.setBlockDimensions(in.getRowsInBlock(), in.getColumnsInBlock());
- break;
-
- case OUTER:
- Identifier id2 = this.getSecondExpr().getOutput();
-
- //check input types and characteristics
- checkNumParameters(3);
- checkMatrixParam(getFirstExpr());
- checkMatrixParam(getSecondExpr());
- checkScalarParam(getThirdExpr());
- checkValueTypeParam(getThirdExpr(), ValueType.STRING);
- if( id.getDim2() > 1 || id2.getDim1()>1 ) {
- raiseValidateError("Outer vector operations require a common dimension of one: " +
- id.getDim1()+"x"+id.getDim2()+" o "+id2.getDim1()+"x"+id2.getDim2()+".", false);
- }
-
- //set output characteristics
- output.setDataType(id.getDataType());
- output.setDimensions(id.getDim1(), id2.getDim2());
- output.setBlockDimensions(id.getRowsInBlock(), id.getColumnsInBlock());
- break;
-
- default:
- if (this.isMathFunction()) {
- // datatype and dimensions are same as this.getExpr()
- if (this.getOpCode() == BuiltinFunctionOp.ABS) {
- output.setValueType(getFirstExpr().getOutput().getValueType());
- } else {
- output.setValueType(ValueType.DOUBLE);
- }
- checkMathFunctionParam();
- output.setDataType(id.getDataType());
- output.setDimensions(id.getDim1(), id.getDim2());
- output.setBlockDimensions(id.getRowsInBlock(), id.getColumnsInBlock());
- } else{
- // always unconditional (because unsupported operation)
- raiseValidateError("Unsupported function "+ this.getOpCode(), false, LanguageErrorCodes.INVALID_PARAMETERS);
- }
- }
- return;
- }
-
- private void expandArguments() {
-
- if ( _args == null ) {
- _args = new Expression[1];
- return;
- }
- Expression [] temp = _args.clone();
- _args = new Expression[_args.length + 1];
- System.arraycopy(temp, 0, _args, 0, temp.length);
- }
-
- @Override
- public boolean multipleReturns() {
- switch(_opcode) {
- case QR:
- case LU:
- case EIGEN:
- return true;
- default:
- return false;
- }
- }
-
- /**
- *
- * @param expr
- * @return
- */
- private boolean isConstant(Expression expr) {
- return ( expr != null && expr instanceof ConstIdentifier );
- }
-
- /**
- *
- * @param expr
- * @return
- * @throws LanguageException
- */
- private double getDoubleValue(Expression expr)
- throws LanguageException
- {
- if ( expr instanceof DoubleIdentifier )
- return ((DoubleIdentifier)expr).getValue();
- else if ( expr instanceof IntIdentifier)
- return ((IntIdentifier)expr).getValue();
- else
- throw new LanguageException("Expecting a numeric value.");
- }
-
- private boolean isMathFunction() {
- switch (this.getOpCode()) {
- case COS:
- case SIN:
- case TAN:
- case ACOS:
- case ASIN:
- case ATAN:
- case SQRT:
- case ABS:
- case LOG:
- case EXP:
- case ROUND:
- case CEIL:
- case FLOOR:
- case MEDIAN:
- return true;
- default:
- return false;
- }
- }
-
- private void checkMathFunctionParam() throws LanguageException {
- switch (this.getOpCode()) {
- case COS:
- case SIN:
- case TAN:
- case ACOS:
- case ASIN:
- case ATAN:
- case SQRT:
- case ABS:
- case EXP:
- case ROUND:
- case CEIL:
- case FLOOR:
- case MEDIAN:
- checkNumParameters(1);
- break;
- case LOG:
- if (getSecondExpr() != null) {
- checkNumParameters(2);
- }
- else {
- checkNumParameters(1);
- }
- break;
- default:
- //always unconditional
- raiseValidateError("Unknown math function "+ this.getOpCode(), false);
- }
- }
-
- public String toString() {
- StringBuilder sb = new StringBuilder(_opcode.toString() + "(" + _args[0].toString());
- for(int i=1; i < _args.length; i++) {
- sb.append(",");
- sb.append(_args[i].toString());
- }
- sb.append(")");
- return sb.toString();
- }
-
- @Override
- // third part of expression IS NOT a variable -- it is the OP to be applied
- public VariableSet variablesRead() {
- VariableSet result = new VariableSet();
-
- for(int i=0; i<_args.length; i++) {
- result.addVariables(_args[i].variablesRead());
- }
-
- return result;
- }
-
- @Override
- public VariableSet variablesUpdated() {
- VariableSet result = new VariableSet();
- // result.addVariables(_first.variablesUpdated());
- return result;
- }
-
- /**
- *
- * @param count
- * @throws LanguageException
- */
- protected void checkNumParameters(int count) //always unconditional
- throws LanguageException
- {
- if (getFirstExpr() == null){
- raiseValidateError("Missing parameter for function "+ this.getOpCode(), false, LanguageErrorCodes.INVALID_PARAMETERS);
- }
-
- if (((count == 1) && (getSecondExpr()!= null || getThirdExpr() != null)) ||
- ((count == 2) && (getThirdExpr() != null))){
- raiseValidateError("Invalid number of parameters for function "+ this.getOpCode(), false, LanguageErrorCodes.INVALID_PARAMETERS);
- }
- else if (((count == 2) && (getSecondExpr() == null)) ||
- ((count == 3) && (getSecondExpr() == null || getThirdExpr() == null))){
- raiseValidateError( "Missing parameter for function "+this.getOpCode(), false, LanguageErrorCodes.INVALID_PARAMETERS);
- }
- }
-
- /**
- *
- * @param e
- * @throws LanguageException
- */
- protected void checkMatrixParam(Expression e) //always unconditional
- throws LanguageException
- {
- if (e.getOutput().getDataType() != DataType.MATRIX) {
- raiseValidateError("Expecting matrix parameter for function "+ this.getOpCode(), false, LanguageErrorCodes.UNSUPPORTED_PARAMETERS);
- }
- }
-
- /**
- *
- * @param e
- * @throws LanguageException
- */
- private void checkScalarParam(Expression e) //always unconditional
- throws LanguageException
- {
- if (e.getOutput().getDataType() != DataType.SCALAR)
- {
- raiseValidateError("Expecting scalar parameter for function " + this.getOpCode(), false, LanguageErrorCodes.UNSUPPORTED_PARAMETERS);
- }
- }
-
- /**
- *
- * @param e
- * @param vt
- * @throws LanguageException
- */
- private void checkValueTypeParam(Expression e, ValueType vt) //always unconditional
- throws LanguageException
- {
- if (e.getOutput().getValueType() != vt) {
- raiseValidateError("Expecting parameter of different value type " + this.getOpCode(), false, LanguageErrorCodes.UNSUPPORTED_PARAMETERS);
- }
- }
-
- private boolean is1DMatrix(Expression e) {
- return (e.getOutput().getDim1() == 1 || e.getOutput().getDim2() == 1 );
- }
-
- private boolean dimsKnown(Expression e) {
- return (e.getOutput().getDim1() != -1 && e.getOutput().getDim2() != -1);
- }
-
- /**
- *
- * @param e
- * @throws LanguageException
- */
- private void check1DMatrixParam(Expression e) //always unconditional
- throws LanguageException
- {
- checkMatrixParam(e);
-
- // throw an exception, when e's output is NOT a one-dimensional matrix
- // the check must be performed only when the dimensions are known at compilation time
- if ( dimsKnown(e) && !is1DMatrix(e)) {
- raiseValidateError("Expecting one-dimensional matrix parameter for function "
- + this.getOpCode(), false, LanguageErrorCodes.UNSUPPORTED_PARAMETERS);
- }
- }
-
- /**
- *
- * @param expr1
- * @param expr2
- * @throws LanguageException
- */
- private void checkMatchingDimensions(Expression expr1, Expression expr2)
- throws LanguageException
- {
- checkMatchingDimensions(expr1, expr2, false);
- }
-
- /**
- *
- * @param expr1
- * @param expr2
- * @throws LanguageException
- */
- private void checkMatchingDimensions(Expression expr1, Expression expr2, boolean allowsMV)
- throws LanguageException
- {
- if (expr1 != null && expr2 != null) {
-
- // if any matrix has unknown dimensions, simply return
- if( expr1.getOutput().getDim1() == -1 || expr2.getOutput().getDim1() == -1
- ||expr1.getOutput().getDim2() == -1 || expr2.getOutput().getDim2() == -1 )
- {
- return;
- }
- else if( (!allowsMV && expr1.getOutput().getDim1() != expr2.getOutput().getDim1())
- || (allowsMV && expr1.getOutput().getDim1() != expr2.getOutput().getDim1() && expr2.getOutput().getDim1() != 1)
- || (!allowsMV && expr1.getOutput().getDim2() != expr2.getOutput().getDim2())
- || (allowsMV && expr1.getOutput().getDim2() != expr2.getOutput().getDim2() && expr2.getOutput().getDim2() != 1) )
- {
- raiseValidateError("Mismatch in matrix dimensions of parameters for function "
- + this.getOpCode(), false, LanguageErrorCodes.INVALID_PARAMETERS);
- }
- }
- }
-
- /**
- *
- * @throws LanguageException
- */
- private void checkMatchingDimensionsQuantile()
- throws LanguageException
- {
- if (getFirstExpr().getOutput().getDim1() != getSecondExpr().getOutput().getDim1()) {
- raiseValidateError("Mismatch in matrix dimensions for "
- + this.getOpCode(), false, LanguageErrorCodes.INVALID_PARAMETERS);
- }
- }
-
- public static BuiltinFunctionExpression getBuiltinFunctionExpression(
- String functionName, ArrayList<ParameterExpression> paramExprsPassed,
- String filename, int blp, int bcp, int elp, int ecp) {
-
- if (functionName == null || paramExprsPassed == null)
- return null;
-
- // check if the function name is built-in function
- // (assign built-in function op if function is built-in
- Expression.BuiltinFunctionOp bifop = null;
-
- if (functionName.equals("avg"))
- bifop = Expression.BuiltinFunctionOp.MEAN;
- else if (functionName.equals("cos"))
- bifop = Expression.BuiltinFunctionOp.COS;
- else if (functionName.equals("sin"))
- bifop = Expression.BuiltinFunctionOp.SIN;
- else if (functionName.equals("tan"))
- bifop = Expression.BuiltinFunctionOp.TAN;
- else if (functionName.equals("acos"))
- bifop = Expression.BuiltinFunctionOp.ACOS;
- else if (functionName.equals("asin"))
- bifop = Expression.BuiltinFunctionOp.ASIN;
- else if (functionName.equals("atan"))
- bifop = Expression.BuiltinFunctionOp.ATAN;
- else if (functionName.equals("diag"))
- bifop = Expression.BuiltinFunctionOp.DIAG;
- else if (functionName.equals("exp"))
- bifop = Expression.BuiltinFunctionOp.EXP;
- else if (functionName.equals("abs"))
- bifop = Expression.BuiltinFunctionOp.ABS;
- else if (functionName.equals("min"))
- bifop = Expression.BuiltinFunctionOp.MIN;
- else if (functionName.equals("max"))
- bifop = Expression.BuiltinFunctionOp.MAX;
- //NOTE: pmin and pmax are just kept for compatibility to R
- // min and max is capable of handling all unary and binary
- // operations (in contrast to R)
- else if (functionName.equals("pmin"))
- bifop = Expression.BuiltinFunctionOp.MIN;
- else if (functionName.equals("pmax"))
- bifop = Expression.BuiltinFunctionOp.MAX;
- else if (functionName.equals("ppred"))
- bifop = Expression.BuiltinFunctionOp.PPRED;
- else if (functionName.equals("log"))
- bifop = Expression.BuiltinFunctionOp.LOG;
- else if (functionName.equals("length"))
- bifop = Expression.BuiltinFunctionOp.LENGTH;
- else if (functionName.equals("ncol"))
- bifop = Expression.BuiltinFunctionOp.NCOL;
- else if (functionName.equals("nrow"))
- bifop = Expression.BuiltinFunctionOp.NROW;
- else if (functionName.equals("sqrt"))
- bifop = Expression.BuiltinFunctionOp.SQRT;
- else if (functionName.equals("sum"))
- bifop = Expression.BuiltinFunctionOp.SUM;
- else if (functionName.equals("mean"))
- bifop = Expression.BuiltinFunctionOp.MEAN;
- else if (functionName.equals("trace"))
- bifop = Expression.BuiltinFunctionOp.TRACE;
- else if (functionName.equals("t"))
- bifop = Expression.BuiltinFunctionOp.TRANS;
- else if (functionName.equals("cbind") || functionName.equals("append"))
- bifop = Expression.BuiltinFunctionOp.CBIND;
- else if (functionName.equals("rbind"))
- bifop = Expression.BuiltinFunctionOp.RBIND;
- else if (functionName.equals("range"))
- bifop = Expression.BuiltinFunctionOp.RANGE;
- else if (functionName.equals("prod"))
- bifop = Expression.BuiltinFunctionOp.PROD;
- else if (functionName.equals("rowSums"))
- bifop = Expression.BuiltinFunctionOp.ROWSUM;
- else if (functionName.equals("colSums"))
- bifop = Expression.BuiltinFunctionOp.COLSUM;
- else if (functionName.equals("rowMins"))
- bifop = Expression.BuiltinFunctionOp.ROWMIN;
- else if (functionName.equals("colMins"))
- bifop = Expression.BuiltinFunctionOp.COLMIN;
- else if (functionName.equals("rowMaxs"))
- bifop = Expression.BuiltinFunctionOp.ROWMAX;
- else if (functionName.equals("rowIndexMax"))
- bifop = Expression.BuiltinFunctionOp.ROWINDEXMAX;
- else if (functionName.equals("rowIndexMin"))
- bifop = Expression.BuiltinFunctionOp.ROWINDEXMIN;
- else if (functionName.equals("colMaxs"))
- bifop = Expression.BuiltinFunctionOp.COLMAX;
- else if (functionName.equals("rowMeans"))
- bifop = Expression.BuiltinFunctionOp.ROWMEAN;
- else if (functionName.equals("colMeans"))
- bifop = Expression.BuiltinFunctionOp.COLMEAN;
- else if (functionName.equals("cummax"))
- bifop = Expression.BuiltinFunctionOp.CUMMAX;
- else if (functionName.equals("cummin"))
- bifop = Expression.BuiltinFunctionOp.CUMMIN;
- else if (functionName.equals("cumprod"))
- bifop = Expression.BuiltinFunctionOp.CUMPROD;
- else if (functionName.equals("cumsum"))
- bifop = Expression.BuiltinFunctionOp.CUMSUM;
- //'castAsScalar' for backwards compatibility
- else if (functionName.equals("as.scalar") || functionName.equals("castAsScalar"))
- bifop = Expression.BuiltinFunctionOp.CAST_AS_SCALAR;
- else if (functionName.equals("as.matrix"))
- bifop = Expression.BuiltinFunctionOp.CAST_AS_MATRIX;
- else if (functionName.equals("as.double"))
- bifop = Expression.BuiltinFunctionOp.CAST_AS_DOUBLE;
- else if (functionName.equals("as.integer"))
- bifop = Expression.BuiltinFunctionOp.CAST_AS_INT;
- else if (functionName.equals("as.logical")) //alternative: as.boolean
- bifop = Expression.BuiltinFunctionOp.CAST_AS_BOOLEAN;
- else if (functionName.equals("quantile"))
- bifop= Expression.BuiltinFunctionOp.QUANTILE;
- else if (functionName.equals("interQuantile"))
- bifop= Expression.BuiltinFunctionOp.INTERQUANTILE;
- else if (functionName.equals("interQuartileMean"))
- bifop= Expression.BuiltinFunctionOp.IQM;
- //'ctable' for backwards compatibility
- else if (functionName.equals("table") || functionName.equals("ctable"))
- bifop = Expression.BuiltinFunctionOp.TABLE;
- else if (functionName.equals("round"))
- bifop = Expression.BuiltinFunctionOp.ROUND;
- //'centralMoment' for backwards compatibility
- else if (functionName.equals("moment") || functionName.equals("centralMoment"))
- bifop = Expression.BuiltinFunctionOp.MOMENT;
- else if (functionName.equals("cov"))
- bifop = Expression.BuiltinFunctionOp.COV;
- else if (functionName.equals("seq"))
- bifop = Expression.BuiltinFunctionOp.SEQ;
- else if (functionName.equals("qr"))
- bifop = Expression.BuiltinFunctionOp.QR;
- else if (functionName.equals("lu"))
- bifop = Expression.BuiltinFunctionOp.LU;
- else if (functionName.equals("eigen"))
- bifop = Expression.BuiltinFunctionOp.EIGEN;
- else if (functionName.equals("solve"))
- bifop = Expression.BuiltinFunctionOp.SOLVE;
- else if (functionName.equals("ceil"))
- bifop = Expression.BuiltinFunctionOp.CEIL;
- else if (functionName.equals("floor"))
- bifop = Expression.BuiltinFunctionOp.FLOOR;
- else if (functionName.equals("median"))
- bifop = Expression.BuiltinFunctionOp.MEDIAN;
- else if (functionName.equals("inv"))
- bifop = Expression.BuiltinFunctionOp.INVERSE;
- else if (functionName.equals("sample"))
- bifop = Expression.BuiltinFunctionOp.SAMPLE;
- else if ( functionName.equals("outer") )
- bifop = Expression.BuiltinFunctionOp.OUTER;
- else
- return null;
-
- BuiltinFunctionExpression retVal = new BuiltinFunctionExpression(bifop, paramExprsPassed,
- filename, blp, bcp, elp, ecp);
-
- return retVal;
- } // end method getBuiltinFunctionExpression
-
- /**
- *
- * @param vt
- * @return
- * @throws LanguageException
- */
- public static BuiltinFunctionOp getValueTypeCastOperator( ValueType vt )
- throws LanguageException
- {
- switch( vt )
- {
- case DOUBLE:
- return BuiltinFunctionOp.CAST_AS_DOUBLE;
- case INT:
- return BuiltinFunctionOp.CAST_AS_INT;
- case BOOLEAN:
- return BuiltinFunctionOp.CAST_AS_BOOLEAN;
- default:
- throw new LanguageException("No cast for value type "+vt);
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/parser/ConditionalPredicate.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/parser/ConditionalPredicate.java b/src/main/java/com/ibm/bi/dml/parser/ConditionalPredicate.java
deleted file mode 100644
index 0b76aaa..0000000
--- a/src/main/java/com/ibm/bi/dml/parser/ConditionalPredicate.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.parser;
-
-public class ConditionalPredicate
-{
-
-
- Expression _expr;
-
-
- public ConditionalPredicate(Expression expr){
- _expr = expr;
- }
-
- public Expression getPredicate(){
- return _expr;
- }
- public void setPredicate(Expression expr){
- _expr = expr;
- }
-
- public String toString(){
- return _expr.toString();
- }
-
-
- public VariableSet variablesRead() {
- VariableSet result = new VariableSet();
- result.addVariables(_expr.variablesRead());
- return result;
- }
-
-
- public VariableSet variablesUpdated() {
- VariableSet result = new VariableSet();
- result.addVariables(_expr.variablesUpdated());
- return result;
- }
-
- ///////////////////////////////////////////////////////////////////////////
- // store position information for expressions
- ///////////////////////////////////////////////////////////////////////////
- private String _filename;
- private int _beginLine, _beginColumn;
- private int _endLine, _endColumn;
-
- public void setFilename(String fname) { _filename = fname; }
- public void setBeginLine(int passed) { _beginLine = passed; }
- public void setBeginColumn(int passed) { _beginColumn = passed; }
- public void setEndLine(int passed) { _endLine = passed; }
- public void setEndColumn(int passed) { _endColumn = passed; }
-
- public void setAllPositions(String fname, int blp, int bcp, int elp, int ecp){
- _filename = fname;
- _beginLine = blp;
- _beginColumn = bcp;
- _endLine = elp;
- _endColumn = ecp;
- }
-
- public String getFilename() { return _filename; }
- public int getBeginLine() { return _beginLine; }
- public int getBeginColumn() { return _beginColumn; }
- public int getEndLine() { return _endLine; }
- public int getEndColumn() { return _endColumn; }
-
- public String printErrorLocation(){
- return "ERROR: line " + _beginLine + ", column " + _beginColumn + " -- ";
- }
-
- public String printWarningLocation(){
- return "WARNING: line " + _beginLine + ", column " + _beginColumn + " -- ";
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/parser/ConstIdentifier.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/parser/ConstIdentifier.java b/src/main/java/com/ibm/bi/dml/parser/ConstIdentifier.java
deleted file mode 100644
index 947484e..0000000
--- a/src/main/java/com/ibm/bi/dml/parser/ConstIdentifier.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.parser;
-
-public abstract class ConstIdentifier extends Identifier
-{
-
- public ConstIdentifier(){
- super();
- }
-
- public abstract long getLongValue()
- throws LanguageException;
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/parser/DMLParseException.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/parser/DMLParseException.java b/src/main/java/com/ibm/bi/dml/parser/DMLParseException.java
deleted file mode 100644
index 8baad0c..0000000
--- a/src/main/java/com/ibm/bi/dml/parser/DMLParseException.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-
-package com.ibm.bi.dml.parser;
-
-import java.util.ArrayList;
-
-public class DMLParseException extends ParseException
-{
-
-
- /**
- * The version identifier for this Serializable class.
- * Increment only if the <i>serialized</i> form of the
- * class changes.
- */
- private static final long serialVersionUID = 1L;
-
- private String _filename;
-
- private ArrayList<DMLParseException> _exceptionList;
-
- public ArrayList<DMLParseException> getExceptionList(){
- return _exceptionList;
- }
-
- public DMLParseException(String fname){
- super();
- _filename = fname;
- _exceptionList = new ArrayList<DMLParseException>();
- }
-
- public DMLParseException(String fname, String msg){
- super(msg);
- _filename = fname;
- _exceptionList = new ArrayList<DMLParseException>();
- _exceptionList.add(this);
- }
-
- public DMLParseException(String fname, Exception e){
- super();
- _filename = fname;
- _exceptionList = new ArrayList<DMLParseException>();
- String newMsg = e.getMessage();
- if (e instanceof ParseException && !(e instanceof DMLParseException)){
- ParseException parseEx = (ParseException)e;
- int beginLine = -1, beginColumn = -1;
- String errorToken = null;
- if (parseEx.currentToken != null){
- beginLine = parseEx.currentToken.beginLine;
- beginColumn = parseEx.currentToken.beginColumn;
- errorToken = parseEx.currentToken.image;
- newMsg = "ERROR: " + _filename + " -- line " + beginLine + ", column " + beginColumn + " -- " + "Parsing error around token \"" + errorToken + "\"";
- } else {
- newMsg = "ERROR: " + _filename + " -- line " + beginLine + ", column " + beginColumn + " -- " + "Parsing error with unspecified token";
- }
- }
-
- _exceptionList.add(new DMLParseException(_filename, newMsg));
- }
-
- public int size(){
- return _exceptionList.size();
- }
-
- public void add(Exception e){
- if (e instanceof DMLParseException)
- _exceptionList.addAll(((DMLParseException)e).getExceptionList());
- else
- _exceptionList.add(new DMLParseException(this._filename, e));
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/parser/DMLProgram.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/parser/DMLProgram.java b/src/main/java/com/ibm/bi/dml/parser/DMLProgram.java
deleted file mode 100644
index 104c231..0000000
--- a/src/main/java/com/ibm/bi/dml/parser/DMLProgram.java
+++ /dev/null
@@ -1,783 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.parser;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.conf.DMLConfig;
-import com.ibm.bi.dml.lops.LopProperties;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.lops.LopsException;
-import com.ibm.bi.dml.lops.compile.Dag;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.ExternalFunctionProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.ExternalFunctionProgramBlockCP;
-import com.ibm.bi.dml.runtime.controlprogram.ForProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.FunctionProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.IfProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.Program;
-import com.ibm.bi.dml.runtime.controlprogram.ProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.WhileProgramBlock;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.ProgramConverter;
-import com.ibm.bi.dml.runtime.instructions.CPInstructionParser;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-
-
-
-public class DMLProgram
-{
-
- private ArrayList<StatementBlock> _blocks;
- private HashMap<String, FunctionStatementBlock> _functionBlocks;
- private HashMap<String,DMLProgram> _namespaces;
- public static String DEFAULT_NAMESPACE = ".defaultNS";
- public static String INTERNAL_NAMESPACE = "_internal"; // used for multi-return builtin functions
- private static final Log LOG = LogFactory.getLog(DMLProgram.class.getName());
-
- public DMLProgram(){
- _blocks = new ArrayList<StatementBlock>();
- _functionBlocks = new HashMap<String,FunctionStatementBlock>();
- _namespaces = new HashMap<String,DMLProgram>();
- _namespaces.put(DMLProgram.DEFAULT_NAMESPACE,this);
- }
-
- public HashMap<String,DMLProgram> getNamespaces(){
- return _namespaces;
- }
-
- public void addStatementBlock(StatementBlock b, int pos) {
- _blocks.add(pos,b) ;
- }
-
- public void addStatementBlock(StatementBlock b){
- _blocks.add(b);
- }
-
- public int getNumStatementBlocks(){
- return _blocks.size();
- }
-
- public FunctionStatementBlock getFunctionStatementBlock(String namespaceKey, String functionName) {
- DMLProgram namespaceProgram = this.getNamespaces().get(namespaceKey);
- if (namespaceProgram == null)
- return null;
-
- // for the namespace DMLProgram, get the specified function (if exists) in its current namespace
- FunctionStatementBlock retVal = namespaceProgram._functionBlocks.get(functionName);
- return retVal;
- }
-
- public HashMap<String, FunctionStatementBlock> getFunctionStatementBlocks(String namespaceKey) throws LanguageException{
- DMLProgram namespaceProgram = this.getNamespaces().get(namespaceKey);
- if (namespaceProgram == null){
- LOG.error("ERROR: namespace " + namespaceKey + " is underfined");
- throw new LanguageException("ERROR: namespace " + namespaceKey + " is underfined");
- }
- // for the namespace DMLProgram, get the functions in its current namespace
- return namespaceProgram._functionBlocks;
- }
-
- public ArrayList<FunctionStatementBlock> getFunctionStatementBlocks()
- throws LanguageException
- {
- ArrayList<FunctionStatementBlock> ret = new ArrayList<FunctionStatementBlock>();
-
- for( DMLProgram nsProg : _namespaces.values() )
- ret.addAll(nsProg._functionBlocks.values());
-
- return ret;
- }
-
- public void addFunctionStatementBlock( String namespace, String fname, FunctionStatementBlock fsb )
- throws LanguageException
- {
- DMLProgram namespaceProgram = this.getNamespaces().get(namespace);
- if (namespaceProgram == null)
- throw new LanguageException( "Namespace does not exist." );
-
- namespaceProgram._functionBlocks.put(fname, fsb);
- }
-
- public ArrayList<StatementBlock> getStatementBlocks(){
- return _blocks;
- }
-
- public void setStatementBlocks(ArrayList<StatementBlock> passed){
- _blocks = passed;
- }
-
- public StatementBlock getStatementBlock(int i){
- return _blocks.get(i);
- }
-
- public void setStatementBlock(int i, StatementBlock sb) {
- _blocks.set(i, sb);
- }
-
- public void mergeStatementBlocks(){
- _blocks = StatementBlock.mergeStatementBlocks(_blocks);
- }
-
- public String toString(){
- StringBuilder sb = new StringBuilder();
-
- // for each namespace, display all functions
- for (String namespaceKey : this.getNamespaces().keySet()){
-
- sb.append("NAMESPACE = " + namespaceKey + "\n");
- DMLProgram namespaceProg = this.getNamespaces().get(namespaceKey);
-
-
- sb.append("FUNCTIONS = ");
-
- for (FunctionStatementBlock fsb : namespaceProg._functionBlocks.values()){
- sb.append(fsb);
- sb.append(", ");
- }
- sb.append("\n");
- sb.append("********************************** \n");
-
- }
-
- sb.append("******** MAIN SCRIPT BODY ******** \n");
- for (StatementBlock b : _blocks){
- sb.append(b);
- sb.append("\n");
- }
- sb.append("********************************** \n");
- return sb.toString();
- }
-
-
- public Program getRuntimeProgram(DMLConfig config) throws IOException, LanguageException, DMLRuntimeException, LopsException, DMLUnsupportedOperationException {
-
- // constructor resets the set of registered functions
- Program rtprog = new Program();
-
- // for all namespaces, translate function statement blocks into function program blocks
- for (String namespace : _namespaces.keySet()){
-
- for (String fname : getFunctionStatementBlocks(namespace).keySet()){
- // add program block to program
- FunctionStatementBlock fsb = getFunctionStatementBlocks(namespace).get(fname);
- FunctionProgramBlock rtpb = (FunctionProgramBlock)createRuntimeProgramBlock(rtprog, fsb, config);
- rtprog.addFunctionProgramBlock(namespace, fname, rtpb);
- rtpb.setRecompileOnce( fsb.isRecompileOnce() );
- }
- }
-
- // for each top-level block
- for (StatementBlock sb : _blocks) {
-
- // add program block to program
- ProgramBlock rtpb = createRuntimeProgramBlock(rtprog, sb, config);
- rtprog.addProgramBlock(rtpb);
- }
-
-
- return rtprog ;
- }
-
- /**
- *
- * @param prog
- * @param sb
- * @param config
- * @return
- * @throws IOException
- * @throws LopsException
- * @throws DMLRuntimeException
- * @throws DMLUnsupportedOperationException
- */
- public ProgramBlock createRuntimeProgramBlock(Program prog, StatementBlock sb, DMLConfig config)
- throws IOException, LopsException, DMLRuntimeException, DMLUnsupportedOperationException
- {
- Dag<Lop> dag = null;
- Dag<Lop> pred_dag = null;
-
- ArrayList<Instruction> instruct;
- ArrayList<Instruction> pred_instruct = null;
-
- ProgramBlock retPB = null;
-
- // process While Statement - add runtime program blocks to program
- if (sb instanceof WhileStatementBlock){
-
- // create DAG for loop predicates
- pred_dag = new Dag<Lop>();
- ((WhileStatementBlock) sb).get_predicateLops().addToDag(pred_dag);
-
- // create instructions for loop predicates
- pred_instruct = new ArrayList<Instruction>();
- ArrayList<Instruction> pInst = pred_dag.getJobs(null, config);
- for (Instruction i : pInst ) {
- pred_instruct.add(i);
- }
-
- // create while program block
- WhileProgramBlock rtpb = new WhileProgramBlock(prog, pred_instruct);
-
- if (rtpb.getPredicateResultVar() == null) {
- // e.g case : WHILE(continue)
- if ( ((WhileStatementBlock) sb).get_predicateLops().getExecLocation() == LopProperties.ExecLocation.Data ) {
- String resultVar = ((WhileStatementBlock) sb).get_predicateLops().getOutputParameters().getLabel();
- rtpb.setPredicateResultVar( resultVar );
- }
- else {
- LOG.error(sb.printBlockErrorLocation() + "Error in translating the WHILE predicate.");
- throw new LopsException(sb.printBlockErrorLocation() + "Error in translating the WHILE predicate.");
-
- }
- }
- //// process the body of the while statement block ////
-
- WhileStatementBlock wsb = (WhileStatementBlock)sb;
- if (wsb.getNumStatements() > 1){
- LOG.error(wsb.printBlockErrorLocation() + "WhileStatementBlock should only have 1 statement");
- throw new LopsException(wsb.printBlockErrorLocation() + "WhileStatementBlock should only have 1 statement");
- }
- WhileStatement wstmt = (WhileStatement)wsb.getStatement(0);
- for (StatementBlock sblock : wstmt.getBody()){
-
- // process the body
- ProgramBlock childBlock = createRuntimeProgramBlock(prog, sblock, config);
- rtpb.addProgramBlock(childBlock);
- }
-
- // check there are actually Lops in to process (loop stmt body will not have any)
- if (wsb.getLops() != null && !wsb.getLops().isEmpty() ){
- LOG.error(wsb.printBlockErrorLocation() + "WhileStatementBlock should have no Lops");
- throw new LopsException(wsb.printBlockErrorLocation() + "WhileStatementBlock should have no Lops");
- }
-
-
- retPB = rtpb;
-
- //post processing for generating missing instructions
- //retPB = verifyAndCorrectProgramBlock(sb.liveIn(), sb.liveOut(), sb._kill, retPB);
-
- // add statement block
- retPB.setStatementBlock(sb);
-
- // add location information
- retPB.setAllPositions(sb.getBeginLine(), sb.getBeginColumn(), sb.getEndLine(), sb.getEndColumn());
- }
-
- // process If Statement - add runtime program blocks to program
- else if (sb instanceof IfStatementBlock){
-
- // create DAG for loop predicates
- pred_dag = new Dag<Lop>();
- ((IfStatementBlock) sb).get_predicateLops().addToDag(pred_dag);
-
- // create instructions for loop predicates
- pred_instruct = new ArrayList<Instruction>();
- ArrayList<Instruction> pInst = pred_dag.getJobs(null, config);
- for (Instruction i : pInst ) {
- pred_instruct.add(i);
- }
-
- // create if program block
- IfProgramBlock rtpb = new IfProgramBlock(prog, pred_instruct);
-
- if (rtpb.getPredicateResultVar() == null ) {
- // e.g case : If(continue)
- if ( ((IfStatementBlock) sb).get_predicateLops().getExecLocation() == LopProperties.ExecLocation.Data ) {
- String resultVar = ((IfStatementBlock) sb).get_predicateLops().getOutputParameters().getLabel();
- rtpb.setPredicateResultVar( resultVar );
- }
- else {
- LOG.error(sb.printBlockErrorLocation() + "Error in translating the IF predicate.");
- throw new LopsException(sb.printBlockErrorLocation() + "Error in translating the IF predicate.");
- }
- }
-
- // process the body of the if statement block
- IfStatementBlock isb = (IfStatementBlock)sb;
- if (isb.getNumStatements() > 1){
- LOG.error(isb.printBlockErrorLocation() + "IfStatementBlock should have only 1 statement");
- throw new LopsException(isb.printBlockErrorLocation() + "IfStatementBlock should have only 1 statement");
- }
- IfStatement istmt = (IfStatement)isb.getStatement(0);
-
- // process the if body
- for (StatementBlock sblock : istmt.getIfBody()){
- ProgramBlock childBlock = createRuntimeProgramBlock(prog, sblock, config);
- rtpb.addProgramBlockIfBody(childBlock);
- }
-
- // process the else body
- for (StatementBlock sblock : istmt.getElseBody()){
- ProgramBlock childBlock = createRuntimeProgramBlock(prog, sblock, config);
- rtpb.addProgramBlockElseBody(childBlock);
- }
-
- // check there are actually Lops in to process (loop stmt body will not have any)
- if (isb.getLops() != null && !isb.getLops().isEmpty() ){
- LOG.error(isb.printBlockErrorLocation() + "IfStatementBlock should have no Lops");
- throw new LopsException(isb.printBlockErrorLocation() + "IfStatementBlock should have no Lops");
- }
-
- retPB = rtpb;
-
- //post processing for generating missing instructions
- //retPB = verifyAndCorrectProgramBlock(sb.liveIn(), sb.liveOut(), sb._kill, retPB);
-
- // add statement block
- retPB.setStatementBlock(sb);
-
- // add location information
- retPB.setAllPositions(sb.getBeginLine(), sb.getBeginColumn(), sb.getEndLine(), sb.getEndColumn());
- }
-
- // process For Statement - add runtime program blocks to program
- // NOTE: applies to ForStatementBlock and ParForStatementBlock
- else if (sb instanceof ForStatementBlock)
- {
- ForStatementBlock fsb = (ForStatementBlock) sb;
-
- // create DAGs for loop predicates
- Dag<Lop> fromDag = new Dag<Lop>();
- Dag<Lop> toDag = new Dag<Lop>();
- Dag<Lop> incrementDag = new Dag<Lop>();
- if( fsb.getFromHops()!=null )
- fsb.getFromLops().addToDag(fromDag);
- if( fsb.getToHops()!=null )
- fsb.getToLops().addToDag(toDag);
- if( fsb.getIncrementHops()!=null )
- fsb.getIncrementLops().addToDag(incrementDag);
-
- // create instructions for loop predicates
- ArrayList<Instruction> fromInstructions = fromDag.getJobs(null, config);
- ArrayList<Instruction> toInstructions = toDag.getJobs(null, config);
- ArrayList<Instruction> incrementInstructions = incrementDag.getJobs(null, config);
-
- // create for program block
- String sbName = null;
- ForProgramBlock rtpb = null;
- IterablePredicate iterPred = fsb.getIterPredicate();
- String [] iterPredData= IterablePredicate.createIterablePredicateVariables(iterPred.getIterVar().getName(),
- fsb.getFromLops(), fsb.getToLops(), fsb.getIncrementLops());
-
- if( sb instanceof ParForStatementBlock )
- {
- sbName = "ParForStatementBlock";
- rtpb = new ParForProgramBlock(prog, iterPredData,iterPred.getParForParams());
- ParForProgramBlock pfrtpb = (ParForProgramBlock)rtpb;
- pfrtpb.setResultVariables( ((ParForStatementBlock)sb).getResultVariables() );
- pfrtpb.setStatementBlock((ParForStatementBlock)sb); //used for optimization and creating unscoped variables
- }
- else //ForStatementBlock
- {
- sbName = "ForStatementBlock";
- rtpb = new ForProgramBlock(prog, iterPredData);
- }
-
- rtpb.setFromInstructions( fromInstructions );
- rtpb.setToInstructions( toInstructions );
- rtpb.setIncrementInstructions( incrementInstructions );
-
- rtpb.setIterablePredicateVars( iterPredData );
-
- // process the body of the for statement block
- if (fsb.getNumStatements() > 1){
- LOG.error(fsb.printBlockErrorLocation() + " " + sbName + " should have 1 statement" );
- throw new LopsException(fsb.printBlockErrorLocation() + " " + sbName + " should have 1 statement" );
- }
- ForStatement fs = (ForStatement)fsb.getStatement(0);
- for (StatementBlock sblock : fs.getBody()){
- ProgramBlock childBlock = createRuntimeProgramBlock(prog, sblock, config);
- rtpb.addProgramBlock(childBlock);
- }
-
- // check there are actually Lops in to process (loop stmt body will not have any)
- if (fsb.getLops() != null && !fsb.getLops().isEmpty()){
- LOG.error(fsb.printBlockErrorLocation() + sbName + " should have no Lops" );
- throw new LopsException(fsb.printBlockErrorLocation() + sbName + " should have no Lops" );
- }
-
- retPB = rtpb;
-
- //post processing for generating missing instructions
- //retPB = verifyAndCorrectProgramBlock(sb.liveIn(), sb.liveOut(), sb._kill, retPB);
-
- // add statement block
- retPB.setStatementBlock(sb);
-
- // add location information
- retPB.setAllPositions(sb.getBeginLine(), sb.getBeginColumn(), sb.getEndLine(), sb.getEndColumn());
- }
-
- // process function statement block - add runtime program blocks to program
- else if (sb instanceof FunctionStatementBlock){
-
- FunctionStatementBlock fsb = (FunctionStatementBlock)sb;
- if (fsb.getNumStatements() > 1){
- LOG.error(fsb.printBlockErrorLocation() + "FunctionStatementBlock should only have 1 statement");
- throw new LopsException(fsb.printBlockErrorLocation() + "FunctionStatementBlock should only have 1 statement");
- }
- FunctionStatement fstmt = (FunctionStatement)fsb.getStatement(0);
- FunctionProgramBlock rtpb = null;
-
- if (fstmt instanceof ExternalFunctionStatement) {
- // create external function program block
-
- String execType = ((ExternalFunctionStatement) fstmt)
- .getOtherParams().get(ExternalFunctionStatement.EXEC_TYPE);
- boolean isCP = (execType.equals(ExternalFunctionStatement.IN_MEMORY)) ? true : false;
-
- String scratchSpaceLoc = null;
- try {
- scratchSpaceLoc = config.getTextValue(DMLConfig.SCRATCH_SPACE);
- } catch (Exception e){
- LOG.error(fsb.printBlockErrorLocation() + "could not retrieve parameter " + DMLConfig.SCRATCH_SPACE + " from DMLConfig");
- }
- StringBuilder buff = new StringBuilder();
- buff.append(scratchSpaceLoc);
- buff.append(Lop.FILE_SEPARATOR);
- buff.append(Lop.PROCESS_PREFIX);
- buff.append(DMLScript.getUUID());
- buff.append(Lop.FILE_SEPARATOR);
- buff.append(ProgramConverter.CP_ROOT_THREAD_ID);
- buff.append(Lop.FILE_SEPARATOR);
- buff.append("PackageSupport");
- buff.append(Lop.FILE_SEPARATOR);
- String basedir = buff.toString();
-
- if( isCP )
- {
-
- rtpb = new ExternalFunctionProgramBlockCP(prog,
- fstmt.getInputParams(), fstmt.getOutputParams(),
- ((ExternalFunctionStatement) fstmt).getOtherParams(),
- basedir );
- }
- else
- {
- rtpb = new ExternalFunctionProgramBlock(prog,
- fstmt.getInputParams(), fstmt.getOutputParams(),
- ((ExternalFunctionStatement) fstmt).getOtherParams(),
- basedir);
- }
-
- if (!fstmt.getBody().isEmpty()){
- LOG.error(fstmt.printErrorLocation() + "ExternalFunctionStatementBlock should have no statement blocks in body");
- throw new LopsException(fstmt.printErrorLocation() + "ExternalFunctionStatementBlock should have no statement blocks in body");
- }
- }
- else
- {
- // create function program block
- rtpb = new FunctionProgramBlock(prog, fstmt.getInputParams(), fstmt.getOutputParams());
-
- // process the function statement body
- for (StatementBlock sblock : fstmt.getBody()){
- // process the body
- ProgramBlock childBlock = createRuntimeProgramBlock(prog, sblock, config);
- rtpb.addProgramBlock(childBlock);
- }
- }
-
- // check there are actually Lops in to process (loop stmt body will not have any)
- if (fsb.getLops() != null && !fsb.getLops().isEmpty()){
- LOG.error(fsb.printBlockErrorLocation() + "FunctionStatementBlock should have no Lops");
- throw new LopsException(fsb.printBlockErrorLocation() + "FunctionStatementBlock should have no Lops");
- }
-
- retPB = rtpb;
-
- // add location information
- retPB.setAllPositions(sb.getBeginLine(), sb.getBeginColumn(), sb.getEndLine(), sb.getEndColumn());
- }
- else {
-
- // handle general case
- ProgramBlock rtpb = new ProgramBlock(prog);
-
- // DAGs for Lops
- dag = new Dag<Lop>();
-
- // check there are actually Lops in to process (loop stmt body will not have any)
- if (sb.getLops() != null && !sb.getLops().isEmpty()){
-
- for (Lop l : sb.getLops()) {
- l.addToDag(dag);
- }
-
- // Instructions for Lobs DAGs
- instruct = dag.getJobs(sb, config);
- rtpb.addInstructions(instruct);
- }
-
- /*// TODO: check with Doug
- // add instruction for a function call
- if (sb.getFunctionCallInst() != null){
- rtpb.addInstruction(sb.getFunctionCallInst());
- }*/
-
- retPB = rtpb;
-
- //post processing for generating missing instructions
- //retPB = verifyAndCorrectProgramBlock(sb.liveIn(), sb.liveOut(), sb._kill, retPB);
-
- // add statement block
- retPB.setStatementBlock(sb);
-
- // add location information
- retPB.setAllPositions(sb.getBeginLine(), sb.getBeginColumn(), sb.getEndLine(), sb.getEndColumn());
- }
-
- return retPB;
- }
-
- /**
- * Post processing of each created program block in order to adhere to livein/liveout
- * (currently needed for cleanup (especially for caching) of intermediate results if the last datasink
- * is an external function because instructions of external functions are created outside hops/lops,
- * e.g., X=..., Y=fun(X) and X is not used afterwards )
- *
- * NOTES:
- * (1) Rule1: checking livein and liveout is sufficient because the last external function is in its own
- * programblock anyway.
- * (2) as we cannot efficiently distinguish if the problematic var is created by an external function
- * or some other instruction, we generate RMVAR instructions although for vars created by non-CP
- * external functions RMFILEVAR instructions are required. However, all remaining files in scratch_space
- * are cleaned after execution anyway.
- * (3) As an alternative to doing rule 2, we could also check for existing objects in createvar and function invocation
- * (or generic at program block level) and remove objects of previous iterations accordingly (but objects of last iteration
- * would still require seperate cleanup).
- *
- * TODO: MB: external function invocations should become hops/lops as well (see instruction gen in DMLTranslator),
- * (currently not possible at Hops/Lops level due the requirement of multiple outputs for functions)
- * TODO: MB: we should in general always leverage livein/liveout during hops/lops generation.
- * TODO: MB: verify and correct can be removed once everything is integrated in hops/lops generation
- *
- * @param in
- * @param out
- * @param pb
- * @return
- * @throws DMLRuntimeException
- * @throws DMLUnsupportedOperationException
- */
- @SuppressWarnings("unused")
- private ProgramBlock verifyAndCorrectProgramBlock(VariableSet in, VariableSet out, VariableSet kill, ProgramBlock pb)
- throws DMLUnsupportedOperationException, DMLRuntimeException
- {
- //RULE 1: if in IN and not in OUT, then there should be an rmvar or rmfilevar inst
- //(currently required for specific cases of external functions)
- for( String varName : in.getVariableNames() )
- if( !out.containsVariable(varName) )
- {
- DataType dt = in.getVariable(varName).getDataType();
- if( !(dt==DataType.MATRIX || dt==DataType.UNKNOWN) )
- continue; //skip rm instructions for non-matrix objects
-
- boolean foundRMInst = rContainsRMInstruction(pb, varName);
-
- if( !foundRMInst )
- {
- //create RMVAR instruction and put it into the programblock
- Instruction inst = createCleanupInstruction(varName);
-
- inst.setLocation(in.getVariable(varName));
-
- addCleanupInstruction(pb, inst);
-
- LOG.trace("Adding instruction (r1) "+inst.toString());
- }
- }
-
- //RULE 2: if in KILL and not in IN and not in OUT, then there should be an rmvar or rmfilevar inst
- //(currently required for specific cases of nested loops)
- for( String varName : kill.getVariableNames() )
- if( (!in.containsVariable(varName)) && (!out.containsVariable(varName)) )
- {
- DataType dt = kill.getVariable(varName).getDataType();
- if( !(dt==DataType.MATRIX || dt==DataType.UNKNOWN) )
- continue; //skip rm instructions for non-matrix objects
-
- boolean foundRMInst = rContainsRMInstruction(pb, varName);
-
- if( !foundRMInst )
- {
- //create RMVAR instruction and put it into the programblock
- Instruction inst = createCleanupInstruction(varName);
-
- inst.setLocation(kill.getVariable(varName));
-
- //System.out.println("add rvar rule2 "+inst.toString());
- addCleanupInstruction(pb, inst);
-
- LOG.trace("Adding instruction (r2) "+inst.toString());
- }
- }
-
- return pb;
- }
-
- private Instruction createCleanupInstruction(String varName)
- throws DMLUnsupportedOperationException, DMLRuntimeException
- {
- //(example "CP+Lops.OPERAND_DELIMITOR+rmvar+Lops.OPERAND_DELIMITOR+Var7")
- StringBuilder sb = new StringBuilder();
- sb.append("CP");
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append("rmvar");
- sb.append(Lop.OPERAND_DELIMITOR);
- sb.append(varName);
- String str = sb.toString();
- Instruction inst = CPInstructionParser.parseSingleInstruction( str );
-
- return inst;
- }
-
- /**
- * Determines if the given program block includes a RMVAR or RMFILEVAR
- * instruction for the given varName.
- *
- * @param pb
- * @param varName
- * @return
- */
- private boolean rContainsRMInstruction(ProgramBlock pb, String varName)
- {
- if (pb instanceof WhileProgramBlock)
- {
- WhileProgramBlock tmp = (WhileProgramBlock)pb;
- for( ProgramBlock c : tmp.getChildBlocks() )
- if( rContainsRMInstruction(c, varName) )
- return true;
- }
- else if (pb instanceof IfProgramBlock)
- {
- IfProgramBlock tmp = (IfProgramBlock)pb;
- for( ProgramBlock c : tmp.getChildBlocksIfBody() )
- if( rContainsRMInstruction(c, varName) )
- return true;
- for( ProgramBlock c : tmp.getChildBlocksElseBody() )
- if( rContainsRMInstruction(c, varName) )
- return true;
- }
- else if (pb instanceof ForProgramBlock) //includes ParFORProgramBlock
- {
- ForProgramBlock tmp = (ForProgramBlock)pb;
- for( ProgramBlock c : tmp.getChildBlocks() )
- if( rContainsRMInstruction(c, varName) )
- return true;
- }
- else if ( pb instanceof FunctionProgramBlock ) //includes ExternalFunctionProgramBlock and ExternalFunctionProgramBlockCP)
- {
- //do nothing
- }
- else
- {
- for( Instruction inst : pb.getInstructions() )
- {
- String instStr = inst.toString();
- if( instStr.contains("rmfilevar"+Lop.OPERAND_DELIMITOR+varName)
- || instStr.contains("rmvar"+Lop.OPERAND_DELIMITOR+varName) )
- {
- return true;
- }
- }
- }
-
-
- return false;
- }
-
- /**
- * Adds the generated cleanup RMVAR instruction to the given program block.
- * In case of generic (last-level) programblocks it is added to the end of
- * the list of instructions, while for complex program blocks it is added to
- * the end of the list of exit instructions.
- *
- * @param pb
- * @param inst
- * @throws DMLRuntimeException
- */
- private void addCleanupInstruction( ProgramBlock pb, Instruction inst )
- throws DMLRuntimeException
- {
- //System.out.println("Adding rm var instructions: "+inst.toString());
-
- if (pb instanceof WhileProgramBlock)
- {
- WhileProgramBlock wpb = (WhileProgramBlock)pb;
- ArrayList<ProgramBlock> childs = wpb.getChildBlocks();
- if( !childs.get(childs.size()-1).getInstructions().isEmpty() ) //generic last level pb
- childs.get(childs.size()-1).addInstruction(inst);
- else{
- ProgramBlock pbNew = new ProgramBlock(pb.getProgram());
- pbNew.addInstruction(inst);
- childs.add(pbNew);
- }
- }
- else if (pb instanceof ForProgramBlock) //includes ParFORProgramBlock
- {
- ForProgramBlock wpb = (ForProgramBlock)pb;
- ArrayList<ProgramBlock> childs = wpb.getChildBlocks();
- if( !childs.get(childs.size()-1).getInstructions().isEmpty() ) //generic last level pb
- childs.get(childs.size()-1).addInstruction(inst);
- else{
- ProgramBlock pbNew = new ProgramBlock(pb.getProgram());
- pbNew.addInstruction(inst);
- childs.add(pbNew);
- }
- }
- else if (pb instanceof IfProgramBlock)
- ((IfProgramBlock)pb).addExitInstruction(inst);
- else if ( pb instanceof FunctionProgramBlock ) //includes ExternalFunctionProgramBlock and ExternalFunctionProgramBlockCP)
- ; //do nothing
- else
- {
- pb.addInstruction(inst); //add inst at end of pb
- }
- }
-
- /**
- *
- * @param fnamespace
- * @param fname
- * @return
- */
- public static String constructFunctionKey(String fnamespace, String fname)
- {
- return fnamespace + Program.KEY_DELIM + fname;
- }
-
- /**
- *
- * @param fkey
- * @return
- */
- public static String[] splitFunctionKey(String fkey)
- {
- return fkey.split(Program.KEY_DELIM);
- }
-}
-