You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by lr...@apache.org on 2015/12/03 19:45:54 UTC
[23/78] [abbrv] [partial] incubator-systemml git commit: Move files
to new package folder structure
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/compile/JobType.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/compile/JobType.java b/src/main/java/com/ibm/bi/dml/lops/compile/JobType.java
deleted file mode 100644
index f8ce6f1..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/compile/JobType.java
+++ /dev/null
@@ -1,252 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops.compile;
-
-import com.ibm.bi.dml.hops.Hop.FileFormatTypes;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.lops.Data;
-import com.ibm.bi.dml.lops.ParameterizedBuiltin;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-
-
-/**
- * This enumeration defines the set of all map-reduce job types. Each job type
- * is associated with following properties:
- *
- * id - Unique identifier.
- *
- * name - Job name.
- *
- * producesIntermediateOutput - set to false if the the job produces an
- * intermediate output that MUST be consumed by subsequent jobs. The
- * intermediate output is NEVER seen by the end user.
- *
- * emptyInputsAllowed - defines whether or not the job can take an empty input
- * file as an input. Currently, this flag is set to true only for RAND jobs.
- *
- * allowsSingleShuffleInstruction - set to true if the job allows only a single
- * instruction in the shuffle phase. For example, jobs that perform matrix
- * multiplication (MMCJ,MMRJ) can perform only one multiplication per job.
- * Allowing multiple multiplications within a single job complicates
- * the implementation (due to specialized key-value pairs for each multiplication)
- * and such a combination can potentially hinder the performance (since these jobs
- * make use of a lot of memory). Similarly, SORT job can sort a single stream of values.
- *
- */
-
-public enum JobType
-{
- /* Add new job types to the following list */
- // (id, name, emptyInputsAllowed, allowsSingleShuffleInstruction, allowsNoOtherInstructions)
- INVALID (-1, "INVALID", false, false, false),
- ANY (0, "ANY", false, false, false),
- GMR (1, "GMR", false, false, false),
- DATAGEN (2, "DATAGEN", true, false, false),
- REBLOCK (3, "REBLOCK", false, false, false),
- MMCJ (4, "MMCJ", false, true, false),
- MMRJ (5, "MMRJ", false, false, false),
- COMBINE (6, "COMBINE", false, false, true),
- SORT (7, "SORT", false, true, true), // allows only "InstructionsBeforeSort" and nothing else.
- CM_COV (8, "CM_COV", false, false, false), // allows only instructions in the mapper
- GROUPED_AGG (9, "GROUPED_AGG", false, false, false),
- //PARTITION (10, "PARTITION", false, false, true), // MB: meta learning removed
- DATA_PARTITION (11, "DATAPARTITION", false, false, true),
- CSV_REBLOCK (12, "CSV_REBLOCK", false, false, false),
- CSV_WRITE (13, "CSV_WRITE", false, false, true),
- TRANSFORM (14, "TRANSFORM", false, true, false),
- GMRCELL (15, "GMRCELL", false, false, false);
-
-
-
- private static int maxJobID = -1;
- static {
- for(JobType jt : JobType.values())
- {
- if(jt.getId() > maxJobID)
- maxJobID = jt.getId();
- }
- }
- /* Following code should not be edited when adding a new job type */
-
- private final int id;
- private final String name;
-
- private final boolean emptyInputsAllowed;
-
- private final boolean allowsSingleShuffleInstruction;
-
- /**
- * Indicates whether a job can piggyback "other" operations.
- * For example, COMBINE job can only piggyback multiple combine operators but can not perform any other operations.
- */
- private final boolean allowsNoOtherInstructions;
-
- JobType(int id, String name, boolean aei, boolean assi, boolean anoi) {
- this.id = id;
- this.name = name;
- this.emptyInputsAllowed = aei;
- this.allowsSingleShuffleInstruction = assi;
- this.allowsNoOtherInstructions = anoi;
- }
-
- public int getId() {
- return id;
- }
-
- public String getName() {
- return name;
- }
-
-/* public boolean producesIntermediateOutput() {
- return producesIntermediateOutput;
- }
-*/
- public boolean areEmptyInputsAllowed() {
- return emptyInputsAllowed;
- }
-
- public boolean allowsSingleShuffleInstruction() {
- return allowsSingleShuffleInstruction;
- }
-
- public boolean allowsNoOtherInstructions() {
- return allowsNoOtherInstructions;
- }
-
- public Lop.Type getShuffleLopType() throws DMLRuntimeException {
- if ( !allowsSingleShuffleInstruction )
- throw new DMLRuntimeException("Shuffle Lop Type is not defined for a job (" + getName() + ") with allowsSingleShuffleInstruction=false.");
- else {
- if ( getName().equals("MMCJ") )
- return Lop.Type.MMCJ;
- else if ( getName().equals("MMRJ") )
- return Lop.Type.MMRJ;
- else if ( getName().equals("SORT") )
- return Lop.Type.SortKeys;
- else if ( getName().equals("TRANSFORM"))
- return Lop.Type.ParameterizedBuiltin;
- else
- throw new DMLRuntimeException("Shuffle Lop Type is not defined for a job (" + getName() + ") that allows a single shuffle instruction.");
- }
- }
-
- public static JobType findJobTypeFromLop(Lop node) {
- Lop.Type lt = node.getType();
- switch(lt) {
- case DataGen: return JobType.DATAGEN;
-
- case ReBlock: return JobType.REBLOCK;
-
- case Grouping: return JobType.GMR;
-
- case MMCJ: return JobType.MMCJ;
-
- case MMRJ: return JobType.MMRJ;
-
- case MMTSJ: return JobType.GMR;
-
- case SortKeys: return JobType.SORT;
-
- case CentralMoment:
- case CoVariance:
- return JobType.CM_COV;
-
- case GroupedAgg: return JobType.GROUPED_AGG;
-
- case CombineBinary:
- case CombineTernary:
- return JobType.COMBINE;
-
- case DataPartition: return JobType.DATA_PARTITION;
-
- case CSVReBlock: return JobType.CSV_REBLOCK;
-
- case ParameterizedBuiltin:
- if( ((ParameterizedBuiltin)node).getOp() == ParameterizedBuiltin.OperationTypes.TRANSFORM )
- return JobType.TRANSFORM;
-
- case Data:
- /*
- * Only Write LOPs with external data formats (except MatrixMarket) produce MR Jobs
- */
- FileFormatTypes fmt = ((Data) node).getFileFormatType();
- if ( fmt == FileFormatTypes.CSV )
- return JobType.CSV_WRITE;
- else
- return null;
-
- default:
- return null;
- }
- }
-
- public boolean isCompatibleWithParentNodes()
- throws DMLRuntimeException
- {
- if ( !allowsSingleShuffleInstruction )
- throw new DMLRuntimeException("isCompatibleWithParentNodes() can not be invoked for a job (" + getName() + ") with allowsSingleShuffleInstruction=false.");
- else {
- if ( getName().equals("MMCJ") )
- return false;
- else if ( getName().equals("MMRJ") || getName().equals("SORT") || getName().equals("TRANSFORM"))
- return true;
- else
- throw new DMLRuntimeException("Implementation for isCompatibleWithParentNodes() is missing for a job (" + getName() + ") that allows a single shuffle instruction.");
- }
- }
-
- public boolean allowsRecordReaderInstructions() {
- if ( getName().equals("GMR") )
- return true;
- else
- return false;
- }
-
- public int getBase() {
- if (id == -1)
- return 0;
- else if (id == 0) {
- // for ANY, return the bit vector with x number of 1's,
- // where x = number of actual job types (i.e., excluding INVALID,ANY)
- //System.out.println("ANY --> " + JobType.values().length + ", " + (Math.pow(2, JobType.values().length-2)-1) + ", " + (Math.pow(2,13-2)-1));
- return (int) Math.pow(2, maxJobID)-1;
- }
- else
- return (int) Math.pow(2, id-1);
- }
-
- public JobType findJobTypeById(int id) {
- for (JobType jt : JobType.values()) {
- if (jt.getId() == id)
- return jt;
- }
- return null;
- }
-
- public JobType findJobTypeByName(String name) {
- for (JobType jt : JobType.values()) {
- if (jt.getName().equalsIgnoreCase(name))
- return jt;
- }
- return null;
- }
-
- public static int getNumJobTypes() {
- return values().length;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/compile/LopComparator.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/compile/LopComparator.java b/src/main/java/com/ibm/bi/dml/lops/compile/LopComparator.java
deleted file mode 100644
index c5099af..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/compile/LopComparator.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops.compile;
-
-import java.util.Comparator;
-
-import com.ibm.bi.dml.lops.Lop;
-
-
-/**
- *
- * Comparator class used in sorting the LopDAG in topological order. Refer to
- * doTopologicalSort_strict_order() in dml/lops/compile/Dag.java
- *
- * Topological sort guarantees the following:
- *
- * 1) All lops with level i appear before any lop with level greater than i
- * (source nodes are at level 0)
- *
- * 2) Within a given level, nodes are ordered by their ID i.e., by the other in
- * which they are created
- *
- * compare() method is designed to respect the above two requirements.
- *
- * @param <N>
- */
-public class LopComparator<N extends Lop>
- implements Comparator<N>
-{
-
- @Override
- public int compare(N o1, N o2) {
- if (o1.getLevel() < o2.getLevel())
- return -1; // o1 is less than o2
- else if (o1.getLevel() > o2.getLevel())
- return 1; // o1 is greater than o2
- else {
- if (o1.getID() < o2.getID())
- return -1; // o1 is less than o2
- else if (o1.getID() > o2.getID())
- return 1; // o1 is greater than o2
- else
- throw new RuntimeException("Unexpected error: ID's of two lops are same.");
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/lops/runtime/RunMRJobs.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/lops/runtime/RunMRJobs.java b/src/main/java/com/ibm/bi/dml/lops/runtime/RunMRJobs.java
deleted file mode 100644
index 804fe2f..0000000
--- a/src/main/java/com/ibm/bi/dml/lops/runtime/RunMRJobs.java
+++ /dev/null
@@ -1,591 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.lops.runtime;
-
-import java.io.IOException;
-
-import org.apache.hadoop.fs.Path;
-import org.apache.wink.json4j.JSONException;
-
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM;
-import com.ibm.bi.dml.conf.ConfigurationManager;
-import com.ibm.bi.dml.conf.DMLConfig;
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.hops.recompile.Recompiler;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.lops.compile.JobType;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.parser.Expression.ValueType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException;
-import com.ibm.bi.dml.runtime.controlprogram.LocalVariableMap;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.parfor.mqo.RuntimePiggybacking;
-import com.ibm.bi.dml.runtime.instructions.MRInstructionParser;
-import com.ibm.bi.dml.runtime.instructions.MRJobInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.Data;
-import com.ibm.bi.dml.runtime.instructions.cp.ScalarObject;
-import com.ibm.bi.dml.runtime.instructions.mr.DataGenMRInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.RandInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.ReblockInstruction;
-import com.ibm.bi.dml.runtime.instructions.mr.SeqInstruction;
-import com.ibm.bi.dml.runtime.io.MatrixWriter;
-import com.ibm.bi.dml.runtime.io.MatrixWriterFactory;
-import com.ibm.bi.dml.runtime.matrix.CMCOVMR;
-import com.ibm.bi.dml.runtime.matrix.CSVReblockMR;
-import com.ibm.bi.dml.runtime.matrix.CombineMR;
-import com.ibm.bi.dml.runtime.matrix.DataGenMR;
-import com.ibm.bi.dml.runtime.matrix.DataPartitionMR;
-import com.ibm.bi.dml.runtime.matrix.GMR;
-import com.ibm.bi.dml.runtime.matrix.GroupedAggMR;
-import com.ibm.bi.dml.runtime.matrix.JobReturn;
-import com.ibm.bi.dml.runtime.matrix.MMCJMR;
-import com.ibm.bi.dml.runtime.matrix.MMRJMR;
-import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
-import com.ibm.bi.dml.runtime.matrix.MatrixDimensionsMetaData;
-import com.ibm.bi.dml.runtime.matrix.MatrixFormatMetaData;
-import com.ibm.bi.dml.runtime.matrix.ReblockMR;
-import com.ibm.bi.dml.runtime.matrix.SortMR;
-import com.ibm.bi.dml.runtime.matrix.WriteCSVMR;
-import com.ibm.bi.dml.runtime.matrix.data.LibMatrixDatagen;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
-import com.ibm.bi.dml.runtime.matrix.data.RandomMatrixGenerator;
-import com.ibm.bi.dml.runtime.transform.DataTransform;
-import com.ibm.bi.dml.runtime.util.MapReduceTool;
-import com.ibm.bi.dml.utils.Statistics;
-
-
-public class RunMRJobs
-{
-
- /**
- * Wrapper for submitting MR job instructions incl preparation and actual submission.
- * The preparation includes (1) pulling stats out of symbol table and populating the
- * instruction, (2) instruction patching, and (3) export of in-memory matrices if
- * required.
- *
- * Furthermore, this wrapper also provides a hook for runtime piggybacking to intercept
- * concurrent job submissions in order to collect and merge instructions.
- *
- * @param inst
- * @param ec
- * @return
- */
-
- public static JobReturn prepareAndSubmitJob( MRJobInstruction inst, ExecutionContext ec )
- throws DMLRuntimeException
- {
- // Obtain references to all input matrices
- MatrixObject[] inputMatrices = inst.extractInputMatrices(ec);
-
- // export dirty matrices to HDFS
- // note: for REBLOCK postponed until we know if necessary
- if( !(inst.getJobType() == JobType.REBLOCK) )
- {
- //export matrices
- for(MatrixObject m : inputMatrices) {
- if ( m.isDirty() || m.getRDDHandle()!=null )
- m.exportData();
- }
-
- //check input files
- checkEmptyInputs( inst, inputMatrices );
- }
-
- // Obtain references to all output matrices
- inst.extractOutputMatrices(ec);
-
- // obtain original state
- String rdInst = inst.getIv_randInstructions();
- String rrInst = inst.getIv_recordReaderInstructions();
- String mapInst = inst.getIv_instructionsInMapper();
- String shuffleInst = inst.getIv_shuffleInstructions();
- String aggInst = inst.getIv_aggInstructions();
- String otherInst = inst.getIv_otherInstructions();
-
- // variable patching (replace placeholders with variables)
- inst.setIv_randInstructions(updateLabels(rdInst, ec.getVariables()));
- inst.setIv_recordReaderInstructions(updateLabels(rrInst, ec.getVariables()));
- inst.setIv_instructionsInMapper(updateLabels(mapInst, ec.getVariables()));
- inst.setIv_shuffleInstructions(updateLabels(shuffleInst, ec.getVariables()));
- inst.setIv_aggInstructions(updateLabels(aggInst, ec.getVariables()));
- inst.setIv_otherInstructions(updateLabels(otherInst, ec.getVariables()));
-
- // runtime piggybacking if applicable
- JobReturn ret = null;
- if( OptimizerUtils.ALLOW_RUNTIME_PIGGYBACKING
- && RuntimePiggybacking.isActive()
- && RuntimePiggybacking.isSupportedJobType(inst.getJobType()) )
- {
- ret = RuntimePiggybacking.submitJob( inst );
- }
- else
- ret = submitJob( inst );
-
- // reset original state
- inst.setIv_randInstructions(rdInst);
- inst.setIv_recordReaderInstructions(rrInst);
- inst.setIv_instructionsInMapper(mapInst);
- inst.setIv_shuffleInstructions(shuffleInst);
- inst.setIv_aggInstructions(aggInst);
- inst.setIv_otherInstructions(otherInst);
-
- return ret;
- }
-
- /**
- * Submits an MR job instruction, without modifying any state of that instruction.
- *
- * @param inst
- * @param ec
- * @return
- * @throws DMLRuntimeException
- */
- public static JobReturn submitJob(MRJobInstruction inst )
- throws DMLRuntimeException
- {
- JobReturn ret = new JobReturn();
- MatrixObject[] inputMatrices = inst.getInputMatrices();
- MatrixObject[] outputMatrices = inst.getOutputMatrices();
- boolean execCP = false;
-
- // Spawn MapReduce Jobs
- try {
- // replace all placeholders in all instructions with appropriate values
- String rdInst = inst.getIv_randInstructions();
- String rrInst = inst.getIv_recordReaderInstructions();
- String mapInst = inst.getIv_instructionsInMapper();
- String shuffleInst = inst.getIv_shuffleInstructions();
- String aggInst = inst.getIv_aggInstructions();
- String otherInst = inst.getIv_otherInstructions();
- boolean jvmReuse = ConfigurationManager.getConfig().getBooleanValue(DMLConfig.JVM_REUSE);
-
- switch(inst.getJobType()) {
-
- case GMR:
- case GMRCELL:
- ret = GMR.runJob(inst, inst.getInputs(), inst.getInputInfos(),
- inst.getRlens(), inst.getClens(), inst.getBrlens(), inst.getBclens(),
- inst.getPartitioned(), inst.getPformats(), inst.getPsizes(),
- rrInst, mapInst, aggInst, otherInst,
- inst.getIv_numReducers(), inst.getIv_replication(), jvmReuse, inst.getIv_resultIndices(), inst.getDimsUnknownFilePrefix(),
- inst.getOutputs(), inst.getOutputInfos() );
- break;
-
- case DATAGEN:
- if( OptimizerUtils.ALLOW_DYN_RECOMPILATION
- && OptimizerUtils.ALLOW_RAND_JOB_RECOMPILE
- && DMLScript.rtplatform != RUNTIME_PLATFORM.HADOOP
- && Recompiler.checkCPDataGen( inst, rdInst ) )
- {
- ret = executeInMemoryDataGenOperations(inst, rdInst, outputMatrices);
- Statistics.decrementNoOfExecutedMRJobs();
- execCP = true;
- }
- else
- {
- ret = DataGenMR.runJob(inst,
- rdInst.split(Lop.INSTRUCTION_DELIMITOR), mapInst, aggInst, otherInst,
- inst.getIv_numReducers(), inst.getIv_replication(), inst.getIv_resultIndices(), inst.getDimsUnknownFilePrefix(),
- inst.getOutputs(), inst.getOutputInfos());
- }
- break;
-
- case CM_COV:
- ret = CMCOVMR.runJob(inst, inst.getInputs(), inst.getInputInfos(),
- inst.getRlens(), inst.getClens(), inst.getBrlens(), inst.getBclens(),
- mapInst, shuffleInst,
- inst.getIv_numReducers(), inst.getIv_replication(), inst.getIv_resultIndices(),
- inst.getOutputs(), inst.getOutputInfos() );
- break;
-
- case GROUPED_AGG:
- ret = GroupedAggMR.runJob(inst, inst.getInputs(), inst.getInputInfos(),
- inst.getRlens(), inst.getClens(), inst.getBrlens(), inst.getBclens(),
- shuffleInst, otherInst,
- inst.getIv_numReducers(), inst.getIv_replication(), inst.getIv_resultIndices(), inst.getDimsUnknownFilePrefix(),
- inst.getOutputs(), inst.getOutputInfos() );
- break;
-
- case REBLOCK:
- case CSV_REBLOCK:
- if( OptimizerUtils.ALLOW_DYN_RECOMPILATION
- && DMLScript.rtplatform != RUNTIME_PLATFORM.HADOOP
- && Recompiler.checkCPReblock( inst, inputMatrices ) )
- {
- ret = executeInMemoryReblockOperations(inst, shuffleInst, inputMatrices, outputMatrices);
- Statistics.decrementNoOfExecutedMRJobs();
- execCP = true;
- }
- else
- {
- // export dirty matrices to HDFS (initially deferred)
- for(MatrixObject m : inputMatrices) {
- if ( m.isDirty() )
- m.exportData();
- }
- checkEmptyInputs( inst, inputMatrices );
-
- if ( inst.getJobType() == JobType.REBLOCK ) {
- ret = ReblockMR.runJob(inst, inst.getInputs(), inst.getInputInfos(),
- inst.getRlens(), inst.getClens(), inst.getBrlens(), inst.getBclens(), getNNZ(inputMatrices),
- mapInst, shuffleInst, otherInst,
- inst.getIv_numReducers(), inst.getIv_replication(), jvmReuse, inst.getIv_resultIndices(),
- inst.getOutputs(), inst.getOutputInfos() );
- }
- else if( inst.getJobType() == JobType.CSV_REBLOCK ) {
- ret = CSVReblockMR.runJob(inst, inst.getInputs(), inst.getInputInfos(),
- inst.getRlens(), inst.getClens(), inst.getBrlens(), inst.getBclens(),
- shuffleInst, otherInst,
- inst.getIv_numReducers(), inst.getIv_replication(), inst.getIv_resultIndices(),
- inst.getOutputs(), inst.getOutputInfos() );
- }
- }
- break;
-
- case CSV_WRITE:
- ret = WriteCSVMR.runJob(inst, inst.getInputs(), inst.getInputInfos(),
- inst.getRlens(), inst.getClens(), inst.getBclens(), inst.getBclens(), shuffleInst,
- inst.getIv_numReducers(), inst.getIv_replication(), inst.getIv_resultIndices(), inst.getOutputs());
- break;
-
- case MMCJ:
- ret = MMCJMR.runJob(inst, inst.getInputs(), inst.getInputInfos(),
- inst.getRlens(), inst.getClens(), inst.getBrlens(), inst.getBclens(),
- mapInst, aggInst, shuffleInst,
- inst.getIv_numReducers(), inst.getIv_replication(),
- inst.getOutputs()[0], inst.getOutputInfos()[0] );
- break;
-
- case MMRJ:
- ret = MMRJMR.runJob(inst, inst.getInputs(), inst.getInputInfos(),
- inst.getRlens(), inst.getClens(), inst.getBrlens(), inst.getBclens(),
- mapInst, aggInst, shuffleInst, otherInst,
- inst.getIv_numReducers(), inst.getIv_replication(), inst.getIv_resultIndices(),
- inst.getOutputs(), inst.getOutputInfos() );
- break;
-
- case SORT:
- boolean weightsflag = true;
- if ( !mapInst.equalsIgnoreCase("") )
- weightsflag = false;
- ret = SortMR.runJob(inst, inst.getInputs()[0], inst.getInputInfos()[0],
- inst.getRlens()[0], inst.getClens()[0], inst.getBrlens()[0], inst.getBclens()[0],
- mapInst, shuffleInst,
- inst.getIv_numReducers(), inst.getIv_replication(),
- inst.getOutputs()[0], inst.getOutputInfos()[0], weightsflag );
- break;
-
- case COMBINE:
- ret = CombineMR.runJob(inst, inst.getInputs(), inst.getInputInfos(),
- inst.getRlens(), inst.getClens(), inst.getBrlens(), inst.getBclens(),
- shuffleInst,
- inst.getIv_numReducers(), inst.getIv_replication(), inst.getIv_resultIndices(),
- inst.getOutputs(), inst.getOutputInfos() );
- break;
-
- case DATA_PARTITION:
- ret = DataPartitionMR.runJob(inst, inputMatrices, shuffleInst, inst.getIv_resultIndices(), outputMatrices, inst.getIv_numReducers(), inst.getIv_replication());
- break;
-
- case TRANSFORM:
-
- if( OptimizerUtils.ALLOW_DYN_RECOMPILATION
- && OptimizerUtils.ALLOW_TRANSFORM_RECOMPILE
- && DMLScript.rtplatform != RUNTIME_PLATFORM.HADOOP
- && Recompiler.checkCPTransform( inst, inputMatrices ) )
- {
- // transform the data and generate output in CSV format
- ret = executeInMemoryTransform(inst, inputMatrices, outputMatrices);
- Statistics.decrementNoOfExecutedMRJobs();
- execCP = true;
- }
- else
- {
- ret = DataTransform.mrDataTransform(inst, inputMatrices, shuffleInst, otherInst, inst.getIv_resultIndices(), outputMatrices, inst.getIv_numReducers(), inst.getIv_replication());
- }
- break;
-
- default:
- throw new DMLRuntimeException("Invalid jobtype: " + inst.getJobType());
- }
-
- } // end of try block
- catch (Exception e) {
- throw new DMLRuntimeException( e );
- }
-
- if (ret.checkReturnStatus()) {
- /*
- * Check if any output is empty. If yes, create a dummy file. Needs
- * to be done only in case of (1) CellOutputInfo and if not CP, or
- * (2) BinaryBlockOutputInfo if not CP and output empty blocks disabled.
- */
- try {
- if( !execCP )
- {
- for (int i = 0; i < outputMatrices.length; i++) {
- //get output meta data
- MatrixFormatMetaData meta = (MatrixFormatMetaData)outputMatrices[i].getMetaData();
- MatrixCharacteristics mc = meta.getMatrixCharacteristics();
- OutputInfo outinfo = meta.getOutputInfo();
- String fname = outputMatrices[i].getFileName();
-
- if (MapReduceTool.isHDFSFileEmpty(fname))
- {
- //prepare output file
- Path filepath = new Path(fname, "0-m-00000");
- MatrixWriter writer = MatrixWriterFactory.createMatrixWriter(outinfo);
- writer.writeEmptyMatrixToHDFS(filepath.toString(), mc.getRows(), mc.getCols(),
- mc.getRowsPerBlock(), mc.getColsPerBlock());
- }
-
- outputMatrices[i].setFileExists(true);
-
- if ( inst.getJobType() != JobType.CSV_WRITE && inst.getJobType() != JobType.TRANSFORM) {
- // write out metadata file
- // Currently, valueType information in not stored in MR instruction,
- // since only DOUBLE matrices are supported ==> hard coded the value type information for now
- MapReduceTool.writeMetaDataFile(fname + ".mtd", ValueType.DOUBLE, ((MatrixDimensionsMetaData)ret.getMetaData(i)).getMatrixCharacteristics(), outinfo);
- }
- }
- }
- return ret;
- } catch (IOException e) {
- throw new DMLRuntimeException(e);
- }
- }
-
- // should not come here!
- throw new DMLRuntimeException("Unexpected Job Type: " + inst.getJobType());
- }
-
- /**
- *
- * @param inst
- * @param inputMatrices
- * @param pb
- * @throws DMLRuntimeException
- */
- private static void checkEmptyInputs( MRJobInstruction inst, MatrixObject[] inputMatrices )
- throws DMLRuntimeException
- {
- // Check if any of the input files are empty.. only for those job types
- // for which empty inputs are NOT allowed
- if (!inst.getJobType().areEmptyInputsAllowed()) {
- for ( int i=0; i < inputMatrices.length; i++ ) {
- try {
- if (MapReduceTool.isHDFSFileEmpty(inputMatrices[i].getFileName())) {
- throw new DMLRuntimeException( "Can not operate on an empty file: " + inputMatrices[i].getFileName());
- }
- } catch (IOException e) {
- throw new DMLRuntimeException( "runtime error occurred -- " , e);
- }
- }
- }
- }
-
- /**
- * Computes the replacement string for a given variable name placeholder string
- * (e.g., ##mVar2## or ##Var5##). The replacement is a HDFS filename for matrix
- * variables, and is the actual value (stored in symbol table) for scalar variables.
- *
- * @param inst
- * @param varName
- * @param map
- * @return
- * @throws DMLRuntimeException
- */
- private static String getVarNameReplacement(String inst, String varName, LocalVariableMap map) throws DMLRuntimeException {
- Data val = map.get(varName);
- if ( val != null ) {
- String replacement = null;
- if (val.getDataType() == DataType.MATRIX) {
- replacement = ((MatrixObject)val).getFileName();
- }
-
- if (val.getDataType() == DataType.SCALAR)
- replacement = "" + ((ScalarObject) val).getStringValue();
- return replacement;
- }
- else {
- throw new DMLRuntimeException("Variable ("+varName+") in Instruction ("+inst+") is not found in the variablemap.");
- }
- }
-
- /**
- * Replaces ALL placeholder strings (such as ##mVar2## and ##Var5##) in a single instruction.
- *
- * @param inst
- * @param map
- * @return
- * @throws DMLRuntimeException
- */
- private static String updateInstLabels(String inst, LocalVariableMap map) throws DMLRuntimeException {
- if ( inst.contains(Lop.VARIABLE_NAME_PLACEHOLDER) ) {
- int skip = Lop.VARIABLE_NAME_PLACEHOLDER.toString().length();
- while ( inst.contains(Lop.VARIABLE_NAME_PLACEHOLDER) ) {
- int startLoc = inst.indexOf(Lop.VARIABLE_NAME_PLACEHOLDER)+skip;
- String varName = inst.substring(startLoc, inst.indexOf(Lop.VARIABLE_NAME_PLACEHOLDER, startLoc));
- String replacement = getVarNameReplacement(inst, varName, map);
- inst = inst.replaceAll(Lop.VARIABLE_NAME_PLACEHOLDER + varName + Lop.VARIABLE_NAME_PLACEHOLDER, replacement);
- }
- }
- return inst;
- }
-
- /**
- * Takes a delimited string of instructions, and replaces ALL placeholder labels
- * (such as ##mVar2## and ##Var5##) in ALL instructions.
- *
- * @param instList
- * @param labelValueMapping
- * @return
- * @throws DMLRuntimeException
- */
- public static String updateLabels (String instList, LocalVariableMap labelValueMapping) throws DMLRuntimeException {
-
- if ( !instList.contains(Lop.VARIABLE_NAME_PLACEHOLDER) )
- return instList;
-
- StringBuilder updateInstList = new StringBuilder();
- String[] ilist = instList.split(Lop.INSTRUCTION_DELIMITOR);
-
- for ( int i=0; i < ilist.length; i++ ) {
- if ( i > 0 )
- updateInstList.append(Lop.INSTRUCTION_DELIMITOR);
-
- updateInstList.append( updateInstLabels(ilist[i], labelValueMapping));
- }
- return updateInstList.toString();
- }
-
-
- /**
- *
- * @param inputMatrices
- * @return
- * @throws DMLRuntimeException
- */
- private static long[] getNNZ( MatrixObject[] inputMatrices )
- throws DMLRuntimeException
- {
- int len = inputMatrices.length;
- long[] ret = new long[len];
- for( int i=0; i<len; i++ )
- {
- MatrixObject mo = inputMatrices[i];
- if( mo != null )
- ret[i] = mo.getNnz();
- else
- ret[i] = -1;
- }
-
- return ret;
- }
-
- /**
- *
- * @param inst
- * @param shuffleInst
- * @param inputMatrices
- * @param outputMatrices
- * @return
- * @throws DMLUnsupportedOperationException
- * @throws DMLRuntimeException
- */
- private static JobReturn executeInMemoryReblockOperations( MRJobInstruction inst, String shuffleInst, MatrixObject[] inputMatrices, MatrixObject[] outputMatrices )
- throws DMLUnsupportedOperationException, DMLRuntimeException
- {
- MatrixCharacteristics[] mc = new MatrixCharacteristics[outputMatrices.length];
- ReblockInstruction[] rblkSet = MRInstructionParser.parseReblockInstructions(shuffleInst);
- byte[] results = inst.getIv_resultIndices();
- for( ReblockInstruction rblk : rblkSet )
- {
- //CP Reblock through caching framework (no copy required: same data, next op copies)
- MatrixBlock mb = inputMatrices[rblk.input].acquireRead();
- for( int i=0; i<results.length; i++ )
- if( rblk.output == results[i] )
- {
- outputMatrices[i].acquireModify( mb );
- outputMatrices[i].release();
- mc[i] = new MatrixCharacteristics(mb.getNumRows(),mb.getNumColumns(), rblk.brlen, rblk.bclen, mb.getNonZeros());
- }
- inputMatrices[rblk.input].release();
- }
-
- return new JobReturn( mc, inst.getOutputInfos(), true);
- }
-
- private static JobReturn executeInMemoryDataGenOperations( MRJobInstruction inst, String randInst, MatrixObject[] outputMatrices )
- throws DMLUnsupportedOperationException, DMLRuntimeException
- {
- MatrixCharacteristics[] mc = new MatrixCharacteristics[outputMatrices.length];
- DataGenMRInstruction[] dgSet = MRInstructionParser.parseDataGenInstructions(randInst);
- byte[] results = inst.getIv_resultIndices();
- for( DataGenMRInstruction ldgInst : dgSet )
- {
- if( ldgInst instanceof RandInstruction )
- {
- //CP Rand block operation
- RandInstruction lrand = (RandInstruction)ldgInst;
- RandomMatrixGenerator rgen = LibMatrixDatagen.createRandomMatrixGenerator(
- lrand.getProbabilityDensityFunction(),
- (int)lrand.getRows(), (int)lrand.getCols(),
- lrand.getRowsInBlock(), lrand.getColsInBlock(),
- lrand.getSparsity(), lrand.getMinValue(), lrand.getMaxValue(),
- lrand.getPdfParams());
-
-
- MatrixBlock mb = MatrixBlock.randOperations(rgen, lrand.getSeed());
-
- for( int i=0; i<results.length; i++ )
- if( lrand.output == results[i] )
- {
- outputMatrices[i].acquireModify( mb );
- outputMatrices[i].release();
- mc[i] = new MatrixCharacteristics(mb.getNumRows(),mb.getNumColumns(), lrand.getRowsInBlock(), lrand.getColsInBlock(), mb.getNonZeros());
- }
- }
- else if( ldgInst instanceof SeqInstruction )
- {
- SeqInstruction lseq = (SeqInstruction) ldgInst;
- MatrixBlock mb = MatrixBlock.seqOperations(lseq.fromValue, lseq.toValue, lseq.incrValue);
- for( int i=0; i<results.length; i++ )
- if( lseq.output == results[i] )
- {
- outputMatrices[i].acquireModify( mb );
- outputMatrices[i].release();
- mc[i] = new MatrixCharacteristics(mb.getNumRows(),mb.getNumColumns(), lseq.getRowsInBlock(), lseq.getColsInBlock(), mb.getNonZeros());
- }
- }
- }
-
- return new JobReturn( mc, inst.getOutputInfos(), true);
- }
-
- private static JobReturn executeInMemoryTransform( MRJobInstruction inst, MatrixObject[] inputMatrices, MatrixObject[] outputMatrices) throws IOException, DMLRuntimeException, IllegalArgumentException, JSONException {
- return DataTransform.cpDataTransform(
- inst.getIv_shuffleInstructions(),
- inputMatrices,
- outputMatrices);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/parser/AParserWrapper.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/parser/AParserWrapper.java b/src/main/java/com/ibm/bi/dml/parser/AParserWrapper.java
deleted file mode 100644
index d392357..0000000
--- a/src/main/java/com/ibm/bi/dml/parser/AParserWrapper.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.parser;
-
-import java.util.HashMap;
-
-import com.ibm.bi.dml.parser.antlr4.DMLParserWrapper;
-import com.ibm.bi.dml.parser.python.PyDMLParserWrapper;
-
-/**
- * Base class for all dml parsers in order to make the various compilation chains
- * independent of the used parser.
- */
-public abstract class AParserWrapper
-{
- /**
- *
- * @param fileName
- * @param dmlScript
- * @param argVals
- * @return
- * @throws ParseException
- */
- public abstract DMLProgram parse(String fileName, String dmlScript, HashMap<String,String> argVals)
- throws ParseException;
-
-
- /**
- * Factory method for creating instances of AParserWrapper, for
- * simplificy fused with the abstract class.
- *
- * @param pydml
- * @return
- */
- public static AParserWrapper createParser(boolean pydml)
- {
- AParserWrapper ret = null;
-
- //create the parser instance
- if( pydml )
- ret = new PyDMLParserWrapper();
- else
- ret = new DMLParserWrapper();
-
- return ret;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/parser/AssignmentStatement.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/parser/AssignmentStatement.java b/src/main/java/com/ibm/bi/dml/parser/AssignmentStatement.java
deleted file mode 100644
index 870cbe3..0000000
--- a/src/main/java/com/ibm/bi/dml/parser/AssignmentStatement.java
+++ /dev/null
@@ -1,146 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.parser;
-
-import java.util.ArrayList;
-
-import com.ibm.bi.dml.api.DMLScript;
-import com.ibm.bi.dml.debug.DMLBreakpointManager;
-
-
-public class AssignmentStatement extends Statement
-{
-
- private ArrayList<DataIdentifier> _targetList;
- private Expression _source;
-
- // rewrites statement to support function inlining (creates deep copy)
- public Statement rewriteStatement(String prefix) throws LanguageException{
-
- // rewrite target (deep copy)
- DataIdentifier newTarget = (DataIdentifier)_targetList.get(0).rewriteExpression(prefix);
-
- // rewrite source (deep copy)
- Expression newSource = _source.rewriteExpression(prefix);
-
- // create rewritten assignment statement (deep copy)
- AssignmentStatement retVal = new AssignmentStatement(newTarget, newSource,this.getBeginLine(),
- this.getBeginColumn(), this.getEndLine(), this.getEndColumn());
-
- return retVal;
- }
-
-
- public AssignmentStatement(DataIdentifier t, Expression s) {
-
- _targetList = new ArrayList<DataIdentifier>();
- _targetList.add(t);
- _source = s;
- }
-
-
- public AssignmentStatement(DataIdentifier t, Expression s, int beginLine, int beginCol, int endLine, int endCol)
- throws LanguageException
- {
- _targetList = new ArrayList<DataIdentifier>();
- _targetList.add(t);
- _source = s;
-
- setBeginLine(beginLine);
- setBeginColumn(beginCol);
- setEndLine(endLine);
- setEndColumn(endCol);
-
- }
-
- public DataIdentifier getTarget(){
- return _targetList.get(0);
- }
-
- public ArrayList<DataIdentifier> getTargetList()
- {
- return _targetList;
- }
-
- public Expression getSource(){
- return _source;
- }
- public void setSource(Expression s){
- _source = s;
- }
-
- @Override
- public boolean controlStatement() {
- // ensure that breakpoints end up in own statement block
- if (DMLScript.ENABLE_DEBUG_MODE) {
- DMLBreakpointManager.insertBreakpoint(_source.getBeginLine());
- return true;
- }
-
- // for now, ensure that function call ends up in different statement block
- if (_source instanceof FunctionCallIdentifier)
- return true;
-
- return false;
- }
-
- public void initializeforwardLV(VariableSet activeIn){
- //do nothing
- }
-
- public VariableSet initializebackwardLV(VariableSet lo){
- return lo;
- }
-
- public VariableSet variablesRead() {
- VariableSet result = new VariableSet();
-
- // add variables read by source expression
- result.addVariables(_source.variablesRead());
-
- // for LHS IndexedIdentifier, add variables for indexing expressions
- for (int i=0; i<_targetList.size(); i++){
- if (_targetList.get(i) instanceof IndexedIdentifier) {
- IndexedIdentifier target = (IndexedIdentifier) _targetList.get(i);
- result.addVariables(target.variablesRead());
- }
- }
- return result;
- }
-
- public VariableSet variablesUpdated() {
- VariableSet result = new VariableSet();
-
- // add target to updated list
- for (DataIdentifier target : _targetList)
- result.addVariable(target.getName(), target);
- return result;
- }
-
- public String toString(){
- StringBuilder sb = new StringBuilder();
- for (int i=0; i< _targetList.size(); i++){
- sb.append(_targetList.get(i).toString());
- }
- sb.append(" = ");
- sb.append(_source.toString());
- sb.append(";");
-
- return sb.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/parser/BinaryExpression.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/parser/BinaryExpression.java b/src/main/java/com/ibm/bi/dml/parser/BinaryExpression.java
deleted file mode 100644
index 38f7fb9..0000000
--- a/src/main/java/com/ibm/bi/dml/parser/BinaryExpression.java
+++ /dev/null
@@ -1,227 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.parser;
-
-import java.util.HashMap;
-
-
-public class BinaryExpression extends Expression
-{
-
- private Expression _left;
- private Expression _right;
- private BinaryOp _opcode;
-
-
- public Expression rewriteExpression(String prefix) throws LanguageException{
-
-
- BinaryExpression newExpr = new BinaryExpression(this._opcode,
- this.getFilename(), this.getBeginLine(), this.getBeginColumn(), this.getEndLine(), this.getEndColumn());
- newExpr.setLeft(_left.rewriteExpression(prefix));
- newExpr.setRight(_right.rewriteExpression(prefix));
- return newExpr;
- }
-
- public BinaryExpression(BinaryOp bop) {
- _kind = Kind.BinaryOp;
- _opcode = bop;
-
- setFilename("MAIN SCRIPT");
- setBeginLine(0);
- setBeginColumn(0);
- setEndLine(0);
- setEndColumn(0);
- }
-
- public BinaryExpression(BinaryOp bop, String filename, int beginLine, int beginColumn, int endLine, int endColumn) {
- _kind = Kind.BinaryOp;
- _opcode = bop;
-
- setFilename(filename);
- setBeginLine(beginLine);
- setBeginColumn(beginColumn);
- setEndLine(endLine);
- setEndColumn(endColumn);
- }
-
-
- public BinaryOp getOpCode() {
- return _opcode;
- }
-
- public void setLeft(Expression l) {
- _left = l;
-
- // update script location information --> left expression is BEFORE in script
- if (_left != null){
- setFilename(_left.getFilename());
- setBeginLine(_left.getBeginLine());
- setBeginColumn(_left.getBeginColumn());
- }
-
- }
-
- public void setRight(Expression r) {
- _right = r;
-
- // update script location information --> right expression is AFTER in script
- if (_right != null){
- setFilename(_right.getFilename());
- setBeginLine(_right.getEndLine());
- setBeginColumn(_right.getEndColumn());
- }
- }
-
- public Expression getLeft() {
- return _left;
- }
-
- public Expression getRight() {
- return _right;
- }
-
- /**
- * Validate parse tree : Process Binary Expression in an assignment
- * statement
- *
- * @throws LanguageException
- */
- @Override
- public void validateExpression(HashMap<String, DataIdentifier> ids, HashMap<String, ConstIdentifier> constVars, boolean conditional)
- throws LanguageException
- {
- //recursive validate
- if (_left instanceof FunctionCallIdentifier || _right instanceof FunctionCallIdentifier){
- raiseValidateError("user-defined function calls not supported in binary expressions",
- false, LanguageException.LanguageErrorCodes.UNSUPPORTED_EXPRESSION);
- }
-
- _left.validateExpression(ids, constVars, conditional);
- _right.validateExpression(ids, constVars, conditional);
-
- //constant propagation (precondition for more complex constant folding rewrite)
- if( _left instanceof DataIdentifier && constVars.containsKey(((DataIdentifier) _left).getName()) )
- _left = constVars.get(((DataIdentifier) _left).getName());
- if( _right instanceof DataIdentifier && constVars.containsKey(((DataIdentifier) _right).getName()) )
- _right = constVars.get(((DataIdentifier) _right).getName());
-
-
- String outputName = getTempName();
- DataIdentifier output = new DataIdentifier(outputName);
- output.setAllPositions(this.getFilename(), this.getBeginLine(), this.getBeginColumn(), this.getEndLine(), this.getEndColumn());
- output.setDataType(computeDataType(this.getLeft(), this.getRight(), true));
- ValueType resultVT = computeValueType(this.getLeft(), this.getRight(), true);
-
- // Override the computed value type, if needed
- if (this.getOpCode() == Expression.BinaryOp.POW
- || this.getOpCode() == Expression.BinaryOp.DIV) {
- resultVT = ValueType.DOUBLE;
- }
-
- output.setValueType(resultVT);
-
- checkAndSetDimensions(output, conditional);
- if (this.getOpCode() == Expression.BinaryOp.MATMULT) {
- if ((this.getLeft().getOutput().getDataType() != DataType.MATRIX) || (this.getRight().getOutput().getDataType() != DataType.MATRIX)) {
- // remove exception for now
- // throw new LanguageException(
- // "Matrix multiplication not supported for scalars",
- // LanguageException.LanguageErrorCodes.INVALID_PARAMETERS);
- }
- if (this.getLeft().getOutput().getDim2() != -1
- && this.getRight().getOutput().getDim1() != -1
- && this.getLeft().getOutput().getDim2() != this.getRight()
- .getOutput().getDim1())
- {
- raiseValidateError("invalid dimensions for matrix multiplication (k1="+this.getLeft().getOutput().getDim2()+", k2="+this.getRight().getOutput().getDim1()+")",
- conditional, LanguageException.LanguageErrorCodes.INVALID_PARAMETERS);
- }
- output.setDimensions(this.getLeft().getOutput().getDim1(), this
- .getRight().getOutput().getDim2());
- }
-
- this.setOutput(output);
- }
-
- private void checkAndSetDimensions(DataIdentifier output, boolean conditional)
- throws LanguageException {
- Identifier left = this.getLeft().getOutput();
- Identifier right = this.getRight().getOutput();
- Identifier pivot = null;
- Identifier aux = null;
-
- if (left.getDataType() == DataType.MATRIX) {
- pivot = left;
- if (right.getDataType() == DataType.MATRIX) {
- aux = right;
- }
- } else if (right.getDataType() == DataType.MATRIX) {
- pivot = right;
- }
-
- if ((pivot != null) && (aux != null))
- {
- //check dimensions binary operations (if dims known)
- if (isSameDimensionBinaryOp(this.getOpCode()) && pivot.dimsKnown() && aux.dimsKnown() )
- {
- if( (pivot.getDim1() != aux.getDim1() && aux.getDim1()>1) //number of rows must always be equivalent if not row vector
- || (pivot.getDim2() != aux.getDim2() && aux.getDim2()>1)) //number of cols must be equivalent if not col vector
- {
- raiseValidateError("Mismatch in dimensions for operation "+ this.toString(), conditional, LanguageException.LanguageErrorCodes.INVALID_PARAMETERS);
- }
- }
- }
-
- //set dimension information
- if (pivot != null) {
- output.setDimensions(pivot.getDim1(), pivot.getDim2());
- }
- }
-
- public String toString() {
-
- return "(" + _left.toString() + " " + _opcode.toString() + " "
- + _right.toString() + ")";
-
- }
-
- @Override
- public VariableSet variablesRead() {
- VariableSet result = new VariableSet();
- result.addVariables(_left.variablesRead());
- result.addVariables(_right.variablesRead());
- return result;
- }
-
- @Override
- public VariableSet variablesUpdated() {
- VariableSet result = new VariableSet();
- result.addVariables(_left.variablesUpdated());
- result.addVariables(_right.variablesUpdated());
- return result;
- }
-
- public static boolean isSameDimensionBinaryOp(BinaryOp op) {
- return (op == BinaryOp.PLUS) || (op == BinaryOp.MINUS)
- || (op == BinaryOp.MULT) || (op == BinaryOp.DIV)
- || (op == BinaryOp.MODULUS) || (op == BinaryOp.INTDIV)
- || (op == BinaryOp.POW);
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/parser/BooleanExpression.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/parser/BooleanExpression.java b/src/main/java/com/ibm/bi/dml/parser/BooleanExpression.java
deleted file mode 100644
index 30f154d..0000000
--- a/src/main/java/com/ibm/bi/dml/parser/BooleanExpression.java
+++ /dev/null
@@ -1,159 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.parser;
-
-import java.util.HashMap;
-
-
-public class BooleanExpression extends Expression
-{
-
- private Expression _left;
- private Expression _right;
- private BooleanOp _opcode;
-
- public BooleanExpression(BooleanOp bop){
- _kind = Kind.BooleanOp;
- _opcode = bop;
-
- setFilename("MAIN SCRIPT");
- setBeginLine(0);
- setBeginColumn(0);
- setEndLine(0);
- setEndColumn(0);
- }
-
- public BooleanExpression(BooleanOp bop, String filename, int beginLine, int beginColumn, int endLine, int endColumn){
- _kind = Kind.BooleanOp;
- _opcode = bop;
-
- setFilename(filename);
- setBeginLine(beginLine);
- setBeginColumn(beginColumn);
- setEndLine(endLine);
- setEndColumn(endColumn);
- }
-
- public BooleanOp getOpCode(){
- return _opcode;
- }
-
- public void setLeft(Expression l){
- _left = l;
-
- // update script location information --> left expression is BEFORE in script
- if (_left != null){
- this.setFilename(_left.getFilename());
- this.setBeginLine(_left.getBeginLine());
- this.setBeginColumn(_left.getBeginColumn());
- }
- }
-
- public void setRight(Expression r){
- _right = r;
-
- // update script location information --> right expression is AFTER in script
- if (_right != null){
- this.setFilename(_right.getFilename());
- this.setBeginLine(_right.getBeginLine());
- this.setBeginColumn(_right.getBeginColumn());
- }
- }
-
- public Expression getLeft(){
- return _left;
- }
-
- public Expression getRight(){
- return _right;
- }
-
- public Expression rewriteExpression(String prefix) throws LanguageException{
-
-
- BooleanExpression newExpr = new BooleanExpression(this._opcode, this.getFilename(), this.getBeginLine(), this.getBeginColumn(), this.getEndLine(), this.getEndColumn());
- newExpr.setLeft(_left.rewriteExpression(prefix));
- if (_right != null)
- newExpr.setRight(_right.rewriteExpression(prefix));
- return newExpr;
- }
-
- /**
- * Validate parse tree : Process Boolean Expression
- */
- @Override
- public void validateExpression(HashMap<String,DataIdentifier> ids, HashMap<String, ConstIdentifier> constVars, boolean conditional) throws LanguageException{
-
- this.getLeft().validateExpression(ids, constVars, conditional);
-
- //recursive validate
- if (_left instanceof FunctionCallIdentifier){
- raiseValidateError("user-defined function calls not supported in boolean expressions",
- false, LanguageException.LanguageErrorCodes.UNSUPPORTED_EXPRESSION);
- }
-
- if (this.getRight() != null) {
-
- if (_right instanceof FunctionCallIdentifier){
- raiseValidateError("user-defined function calls not supported in boolean expressions",
- false, LanguageException.LanguageErrorCodes.UNSUPPORTED_EXPRESSION);
- }
-
- this.getRight().validateExpression(ids, constVars, conditional);
- }
- String outputName = getTempName();
- DataIdentifier output = new DataIdentifier(outputName);
- output.setAllPositions(this.getFilename(), this.getBeginLine(), this.getBeginColumn(), this.getEndLine(), this.getEndColumn());
-
- output.setBooleanProperties();
- this.setOutput(output);
- if ((_opcode == Expression.BooleanOp.CONDITIONALAND) ||
- (_opcode == Expression.BooleanOp.CONDITIONALOR))
- { //always unconditional (because unsupported operation)
- raiseValidateError("Unsupported boolean operation " + _opcode.toString(), false, LanguageException.LanguageErrorCodes.UNSUPPORTED_PARAMETERS);
- }
- }
-
- public String toString(){
- if (_opcode == BooleanOp.NOT) {
- return "(" + _opcode.toString() + " " + _left.toString() + ")";
- } else {
- return "(" + _left.toString() + " " + _opcode.toString() + " " + _right.toString() + ")";
- }
- }
-
- @Override
- public VariableSet variablesRead() {
- VariableSet result = new VariableSet();
- result.addVariables(_left.variablesRead());
- if (_right != null){
- result.addVariables(_right.variablesRead());
- }
- return result;
- }
-
- @Override
- public VariableSet variablesUpdated() {
- VariableSet result = new VariableSet();
- result.addVariables(_left.variablesUpdated());
- if (_right != null){
- result.addVariables(_right.variablesUpdated());
- }
- return result;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/parser/BooleanIdentifier.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/parser/BooleanIdentifier.java b/src/main/java/com/ibm/bi/dml/parser/BooleanIdentifier.java
deleted file mode 100644
index ac5cb92..0000000
--- a/src/main/java/com/ibm/bi/dml/parser/BooleanIdentifier.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.parser;
-
-
-
-public class BooleanIdentifier extends ConstIdentifier
-{
-
- private boolean _val;
-
-
- public BooleanIdentifier(boolean val, String filename, int blp, int bcp, int elp, int ecp){
- super();
- _val = val;
- _kind = Kind.Data;
- this.setDimensions(0,0);
- this.computeDataType();
- this.setValueType(ValueType.BOOLEAN);
- this.setAllPositions(filename, blp, bcp, elp, ecp);
- }
-
- public Expression rewriteExpression(String prefix) throws LanguageException{
- return this;
- }
-
- public boolean getValue(){
- return _val;
- }
-
- public String toString(){
- return Boolean.toString(_val);
- }
-
- @Override
- public VariableSet variablesRead() {
- return null;
- }
-
- @Override
- public VariableSet variablesUpdated() {
- return null;
- }
-
- @Override
- public long getLongValue(){
- return (getValue() ? 1 : 0);
- }
-}